Browse Source

Added lastupdate HTTP handler.

Added noattinfo GET parameter for schedule.json .
Changed DaySchedule.Start/End to contain the seconds since start of day,
and not the full unixtime.
Added DaySchedule.BaseUts which contains the unixtime of the start of
the day.
Added lid GET paramater to attende.json, which fetches all attendees
with that lid (location id).
Fixed (hopefully) time zone stuff.
Added nocache GET paramater to schedule.json, weeks.json,
lastupdate.json with which cache will not be looked up (but is saved).
Nocache will only work for IP addresses whitelisted (can be set in
config file).
master
Remi Reuvekamp 7 years ago
parent
commit
d8258c65b0
  1. 14
      handlers/attendee.go
  2. 25
      handlers/helpers.go
  3. 52
      handlers/lastupdate.go
  4. 11
      handlers/weeks.go
  5. 14
      handlers/wsched.go
  6. 98
      lastupdate/cache.go
  7. 122
      lastupdate/lastupdate.go
  8. 19
      main.go
  9. 6
      misc/misc.go
  10. 6
      weeks/cache.go
  11. 20
      weeks/weeks.go
  12. 2
      weekschedule/formats.go
  13. 84
      weekschedule/weekschedule.go

14
handlers/attendee.go

@ -36,12 +36,20 @@ func Attendee(w http.ResponseWriter, r *http.Request) {
}
}
if idsStr == "" {
writeJSON(w, r, errStr{Error: "invalid aid/ids (attendee id)"}, time.Time{})
lid, _ := strconv.Atoi(r.FormValue("lid"))
var sqlStr string
switch true {
case idsStr != "":
sqlStr = "WHERE id IN (" + idsStr + ")"
case lid > 0:
sqlStr = "WHERE lid = " + strconv.Itoa(lid)
default:
writeJSON(w, r, errStr{Error: "invalid aid/ids (attendee id) and lid (location id)"}, time.Time{})
return
}
atts, err := attendee.FetchS([]string{"id", "name", "type"}, "WHERE id IN ("+idsStr+")")
atts, err := attendee.FetchS([]string{"id", "name", "type"}, sqlStr)
if err != nil {
writeJSON(w, r, errStr{Error: "error fetching attendees"}, time.Time{})

25
handlers/helpers.go

@ -3,10 +3,13 @@ package handlers
import (
"encoding/json"
"errors"
"net"
"net/http"
"strconv"
"strings"
"time"
"github.com/rreuvekamp/xedule-api/misc"
)
type errStr struct {
@ -48,3 +51,25 @@ func writeJSON(w http.ResponseWriter, r *http.Request, v interface{}, tm time.Ti
_, err = w.Write(data)
return err
}
// ip returns the (correct) Ip address for the given http.Request.
func ip(r *http.Request) string {
ip, _, _ := net.SplitHostPort(r.RemoteAddr)
// Proxy stuff
if ip == "127.0.0.1" {
ip = r.Header.Get("X-FORWARDED-FOR")
}
return ip
}
// checkCacheWhitelist returns a boolean which is true if the given IP address is
// whitelisted to make API requests of which cache is not looked up.
func checkCacheWhitelist(addr string) bool {
// NoCache only has effect if the remoteaddr is whitelisted in the config file.
for _, a := range misc.Cfg().Api.NoCacheIpAllow {
if a == addr {
return true
}
}
return false
}

52
handlers/lastupdate.go

@ -0,0 +1,52 @@
package handlers
import (
"net/http"
"strconv"
"time"
"github.com/rreuvekamp/xedule-api/lastupdate"
)
type pageLu struct {
Year int `json:"year"`
Week int `json:"week"`
Uts int64 `json:"uts"`
}
func LastUpdate(w http.ResponseWriter, r *http.Request) {
yr, wk := time.Now().ISOWeek()
if year, err := strconv.Atoi(r.FormValue("year")); err == nil {
yr = year
}
if week, err := strconv.Atoi(r.FormValue("week")); err == nil {
wk = week
}
cache := true
// NoCache only has effect if the remoteaddr is whitelisted in the config file.
if r.FormValue("nocache") != "" && checkCacheWhitelist(ip(r)) {
cache = false
}
tm, tmCache, _ := lastupdate.Get(yr, wk, cache)
if tm == (time.Time{}) {
p := errStr{
Error: "time for year/week not found",
}
writeJSON(w, r, p, time.Time{})
return
}
p := pageLu{
Year: yr,
Week: wk,
Uts: tm.Unix(),
}
writeJSON(w, r, p, tmCache)
}

11
handlers/weeks.go

@ -7,6 +7,15 @@ import (
)
func Weeks(w http.ResponseWriter, r *http.Request) {
wks, tm, _ := weeks.Get()
cache := true
// NoCache only has effect if the remoteaddr is whitelisted in the config file.
if r.FormValue("nocache") != "" && checkCacheWhitelist(ip(r)) {
cache = false
}
wks, tm, _ := weeks.Get(cache)
writeJSON(w, r, wks, tm)
}

14
handlers/wsched.go

@ -36,7 +36,19 @@ func WSched(w http.ResponseWriter, r *http.Request) {
week = cWeek
}
wk, tm, _ := wsched.Get(aid, year, week)
includeAttsInfo := true
cache := true
if r.FormValue("noattinfo") != "" {
includeAttsInfo = false
}
// NoCache only has effect if the remoteaddr is whitelisted in the config file.
if r.FormValue("nocache") != "" && checkCacheWhitelist(ip(r)) {
cache = false
}
wk, tm, _ := wsched.Get(aid, year, week, includeAttsInfo, cache)
if r.FormValue("legacy") != "" {
writeJSON(w, r, wk.Legacy(), tm)

98
lastupdate/cache.go

@ -0,0 +1,98 @@
package lastupdate
import (
"fmt"
"strconv"
"time"
"github.com/rreuvekamp/xedule-api/attendee"
)
type cacheReq struct {
ch chan cacheResp
year int
week int
}
type cacheResp struct {
found bool
tmLu time.Time
tm time.Time // Cache time
}
type cacheAdd struct {
tmLu time.Time
year int
week int
tm time.Time // Cache time
}
type cacheItem struct {
tmLu time.Time
year int
week int
tm time.Time // Cache time
}
var lastupdates = make(map[int]cacheItem)
var att attendee.Attendee
var cleanMaxAge = 10 * time.Minute
var chLuReq = make(chan cacheReq, 1) // Request (/check for) cache
var chLuAdd = make(chan cacheAdd, 1) // Add cache
var chAttReq = make(chan chan attendee.Attendee)
// Run inits cache and handles cache requests.
func Run() {
// Get the first attendee in the database.
// Used for fetching the weeks list in Get.
a, err := attendee.FetchS([]string{"id", "name", "type", "lid"},
"ORDER BY ID LIMIT 1")
if err == nil || len(a) > 0 {
att = a[0]
}
clean := time.NewTicker(10 * time.Minute)
for {
select {
case r := <-chLuReq: // Request (/lookup) cache
ci, ok := lastupdates[luId(r.year, r.week)]
var re cacheResp
if !ok {
r.ch <- re
continue
}
re.found = true
re.tmLu = ci.tmLu
re.tm = ci.tm // Cache time
r.ch <- re
case r := <-chLuAdd: // Add/update cache
lastupdates[luId(r.year, r.week)] = cacheItem{tmLu: r.tmLu, year: r.year, week: r.week, tm: r.tm}
case ch := <-chAttReq:
ch <- att
case <-clean.C: // Periodicly clean up cache
// Check for outdated cache items, and remove them from the map.
var removes []int
for id, ci := range lastupdates {
if time.Since(ci.tm).Seconds() > cleanMaxAge.Seconds() {
delete(lastupdates, id)
removes = append(removes, id)
}
}
if len(removes) > 0 {
fmt.Println("LastUpdates cache cleaned:", len(removes), removes)
}
}
}
}
// luId makes an id used in the cache map by year/week.
func luId(year, week int) int {
id, _ := strconv.Atoi(strconv.Itoa(year) + strconv.Itoa(week))
return id
}

122
lastupdate/lastupdate.go

@ -0,0 +1,122 @@
package lastupdate
import (
"fmt"
"log"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/rreuvekamp/xedule-api/attendee"
"github.com/rreuvekamp/xedule-api/misc"
"golang.org/x/net/html"
)
const urlLastUpdate = "%sAttendee/ChangeWeek/%d?Code=henk&attId=%d&OreId=%d"
const tmLayout = "2-1-2006 15:04:05"
func Get(year, week int, cache bool) (time.Time, time.Time, error) {
if cache {
// Check for cache
ch := make(chan cacheResp)
chLuReq <- cacheReq{
ch: ch,
year: year,
week: week,
}
c := <-ch
// Serve cache if it exists.
if c.found {
fmt.Println("Found cache")
return c.tmLu, c.tm, nil
}
}
// Get the attendee for fetching the weeks list.
chA := make(chan attendee.Attendee)
chAttReq <- chA
a := <-chA
// Fetch page.
resp, err := http.PostForm(fmt.Sprintf(urlLastUpdate, misc.UrlPrefix, a.Id, a.Type, a.Lid),
url.Values{"currentWeek": {strconv.Itoa(year) + "/" + strconv.Itoa(week)}})
if err != nil {
log.Println("ERROR fetching page with last update:", err, year, week, a)
return time.Time{}, time.Time{}, err
}
defer resp.Body.Close()
// Parse page
doc, err := html.Parse(resp.Body)
if err != nil {
log.Println("ERROR parsing fetched last update:", err, year, week, a)
}
tmLu, _ := parse(doc)
// Save cache
chLuAdd <- cacheAdd{
tmLu: tmLu,
year: year,
week: week,
tm: time.Now(),
}
return tmLu, time.Time{}, err
}
func parse(n *html.Node) (time.Time, bool) {
if n.Type == html.ElementNode && n.Data == "div" {
var correct bool
for _, a := range n.Attr {
if a.Key == "class" && a.Val == "dateCreated" {
correct = true
break
}
}
if !correct {
goto next
}
c := n.FirstChild
if c == nil {
goto next
}
split := strings.Split(c.Data, ":\n")
if len(split) < 2 {
log.Println("WARNING/ERROR: Len of splitted data of lastupdate is < 2:", split)
return time.Time{}, true
}
str := strings.TrimSpace(split[1])
tm, err := time.ParseInLocation(tmLayout, str, misc.Loc)
if err != nil {
log.Println("ERROR when parsing time of fetched lastupdate:", err)
return time.Time{}, true
}
fmt.Println(tm.Unix())
return tm, true
}
next:
for c := n.FirstChild; c != nil; c = c.NextSibling {
tm, done := parse(c)
if done {
return tm, done
}
}
return time.Time{}, false
}

19
main.go

@ -11,6 +11,7 @@ import (
"github.com/rreuvekamp/xedule-api/attendee"
"github.com/rreuvekamp/xedule-api/handlers"
"github.com/rreuvekamp/xedule-api/lastupdate"
"github.com/rreuvekamp/xedule-api/misc"
"github.com/rreuvekamp/xedule-api/weeks"
"github.com/rreuvekamp/xedule-api/weekschedule"
@ -18,10 +19,14 @@ import (
/*
To do:
Have attendees be in memory instead of database.
Log HTTP requests
/attendee.json?aid=14327,14309
+/schedule.json also giving list of attendee ids
+/schedule.json&aid=14307&noattinfo=true
+WeekSchedule.BaseUts // UnixTimeStamp of the first second in the week
+event.Start, event.End <- No Uts but amount of seconds since start of day.
+attendee.json?lid=34
+Fixed time zone stuff
+lastupdate.json?year=2015&week=23
+?nocache=true // Does not look for cache, but does update it of course.
lastupdate and weeks have got their own attendee in cache, can't they share it?
*/
func main() {
@ -30,7 +35,7 @@ func main() {
os.Exit(1)
}
// Don't exit program. Without a database this application can still
// Don't exit program on error. Without a database this application can still
// preform some tasks (WeekSchedule without attendee types).
misc.ConnectDb()
@ -49,11 +54,13 @@ func main() {
go wsched.RunCache()
go weeks.Run()
go lastupdate.Run()
http.HandleFunc("/schedule.json", handlers.WSched)
http.HandleFunc("/weeks.json", handlers.Weeks)
http.HandleFunc("/attendee.json", handlers.Attendee)
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
http.HandleFunc("/lastupdate.json", handlers.LastUpdate)
http.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {
w.Write([]byte("https://github.com/rreuvekamp/xedule-api"))
})

6
misc/misc.go

@ -7,6 +7,7 @@ import (
"fmt"
"io/ioutil"
"log"
"time"
_ "github.com/ziutek/mymysql/godrv"
)
@ -20,6 +21,9 @@ type config struct {
Pass string `json:"password"`
Db string `json:"database"`
} `json:"database"`
Api struct {
NoCacheIpAllow []string `json:"noCacheIpWhitelist"`
} `json:"api"`
}
const CfgFilename = "config.json"
@ -28,6 +32,8 @@ const UrlPrefix = "https://summacollege.xedule.nl/"
var cfg config
var Db *sql.DB
var Loc, _ = time.LoadLocation("Europe/Amsterdam")
// ConnectDb initializes the database 'instance'.
func ConnectDb() error {
var err error

6
weeks/cache.go

@ -17,18 +17,18 @@ type weeksResponse struct {
time time.Time
}
type cache struct {
type weeksCache struct {
wks weeks
time time.Time
}
var defReqMaxAge = time.Minute * 30
var cacheWeeks cache
var cacheWeeks weeksCache
var att attendee.Attendee
var chWksReq = make(chan weeksRequest, 1) // Request weeks in cache
var chWksSet = make(chan cache, 1) // Set weeks in cache
var chWksSet = make(chan weeksCache, 1) // Set weeks in cache
var chAttReq = make(chan chan attendee.Attendee, 1) // Request attendee for Get
func Run() {

20
weeks/weeks.go

@ -21,14 +21,16 @@ const urlWeeks = "%sAttendee/ScheduleCurrent/%d?Code=henk&attId=%d&OreId=%d" //
// Get either fetches the weekslist from an external page (see urlWeeks) and returns is
// or returns the list from cache, if the cache is valid (and not out dated).
func Get() (weeks, time.Time, error) {
// Check for valid cache, return that if valid.
ch := make(chan weeksResponse)
chWksReq <- weeksRequest{ch: ch, maxAge: defReqMaxAge}
wks := <-ch
if wks.found {
return wks.wks, wks.time, nil
func Get(cache bool) (weeks, time.Time, error) {
if cache {
// Check for valid cache, return that if valid.
ch := make(chan weeksResponse)
chWksReq <- weeksRequest{ch: ch, maxAge: defReqMaxAge}
wks := <-ch
if wks.found {
return wks.wks, wks.time, nil
}
}
// Get the attendee for fetching the weeks list.
@ -57,7 +59,7 @@ func Get() (weeks, time.Time, error) {
sort.Sort(sort.Reverse(w))
// Update the cache
chWksSet <- cache{wks: w, time: time.Now()}
chWksSet <- weeksCache{wks: w, time: time.Now()}
return w, time.Now(), nil
}

2
weekschedule/formats.go

@ -25,7 +25,7 @@ type legacyEvent struct {
const legacyDate = "Mon Jan 02 2006"
const legacyTime = "15:04"
var legacyTimeAdd = time.Duration(time.Hour * 2)
var legacyTimeAdd = time.Duration(time.Hour * 0)
// Legacy formats the WeekSchedule in a []legacyday.
func (w WeekSchedule) Legacy() []legacyDay {

84
weekschedule/weekschedule.go

@ -29,19 +29,17 @@ type WeekSchedule struct {
// DaySchedule contains all Events of an attendee for a day.
type DaySchedule struct {
Day time.Weekday `json:"day"`
Events []Event `json:"events"`
Day time.Weekday `json:"day"`
BaseUts int64 `json:"baseuts"`
Events []Event `json:"events"`
}
// Event is a single event for an attendee.
type Event struct {
Start int64 `json:"start"`
End int64 `json:"end"`
Start int `json:"start"`
End int `json:"end"`
Desc string `json:"desc"` // Description
Atts []int `json:"atts"`
/*Classes []string `json:"classes,omitempty"` // (Other) classes/attendees
Facs []string `json:"facs,omitempty"` // Facilities
Staffs []string `json:"staffs,omitempty"`*/
// Used by Fetch
// Holds the attendee names before there are looked up and put in Atts.
@ -51,31 +49,37 @@ type Event struct {
end time.Time
}
const icsTimeLayout = "20060102T150405Z"
const icsTimeLayout = "20060102T150405"
const urlWSched = "%sCalendar/iCalendarICS/%d?year=%d&week=%d"
// Get either returns the WeekSchedule for the given aid, year and week from cache
// or if no valid cache, fetches the ICS file, parses it and returns the WeekSchedule from that.
func Get(aid, year, week int) (WeekSchedule, time.Time, error) {
// Request cache
ch := make(chan cacheResponse)
chWkReq <- cacheRequest{
ch: ch,
aid: aid,
year: year,
week: week,
maxAge: defReqMaxAge,
}
// When includeAttsInfo is set to true, Weekschedule.Atts will be occupied with attendees that are
// in one or more event.
func Get(aid, year, week int, includeAttsInfo, cache bool) (WeekSchedule, time.Time, error) {
if cache {
// Request cache
ch := make(chan cacheResponse)
chWkReq <- cacheRequest{
ch: ch,
aid: aid,
year: year,
week: week,
maxAge: defReqMaxAge,
}
// Wait for and handle cache response
c := <-ch
if c.found {
return c.w, c.time, nil
}
// Wait for and handle cache response
c := <-ch
if c.found {
if !includeAttsInfo && len(c.w.Atts) > 0 {
c.w.Atts = nil
}
// Check cache
// Serve cache if not outdated.
return c.w, c.time, nil
}
}
resp, err := http.Get(fmt.Sprintf(urlWSched, misc.UrlPrefix, aid, year, week))
if err != nil {
@ -94,6 +98,7 @@ func Get(aid, year, week int) (WeekSchedule, time.Time, error) {
var cur Event
var days []DaySchedule
var atts []string // Slice of attendee names which type should be looked up.
var baseUts int64
// The parsing itself
loop:
@ -108,20 +113,26 @@ loop:
}
// Clean/reset current event.
cur = Event{}
case "DTSTART": // DTEND:20150428T090000Z
cur.start, err = time.Parse(icsTimeLayout, strings.TrimSpace(icsIndex(split, 1)))
case "DTSTART": // DTEND:20150428T090000
cur.start, err = time.ParseInLocation(icsTimeLayout, strings.TrimSpace(icsIndex(split, 1)), misc.Loc)
if err != nil {
log.Println("ERROR parsing start time of ICS: \n", err, split)
continue loop
}
cur.Start = cur.start.Unix()
case "DTEND": // DTSTART:20150428T073000Z
cur.end, err = time.Parse(icsTimeLayout, strings.TrimSpace(icsIndex(split, 1)))
//if baseUts == 0 {
baseUts = time.Date(cur.start.Year(), cur.start.Month(), cur.start.Day(), 0, 0, 0, 0, misc.Loc).Unix()
//}
cur.Start = int(cur.start.Unix() - baseUts)
case "DTEND": // DTSTART:20150428T073000
cur.end, err = time.ParseInLocation(icsTimeLayout, strings.TrimSpace(icsIndex(split, 1)), misc.Loc)
if err != nil {
log.Println("ERROR parsing end time of ICS: \n", err, split)
continue loop
}
cur.End = cur.end.Unix()
//if baseUts == 0 {
baseUts = time.Date(cur.end.Year(), cur.end.Month(), cur.end.Day(), 0, 0, 0, 0, misc.Loc).Unix()
//}
cur.End = int(cur.end.Unix() - baseUts)
case "DESCRIPTION": // DESCRIPTION:test
desc := icsIndex(split, 1)
if len(desc) != 0 {
@ -150,8 +161,9 @@ loop:
}
if !success {
days = append(days, DaySchedule{
Events: []Event{cur},
Day: cur.start.Weekday(),
Events: []Event{cur},
BaseUts: baseUts,
Day: cur.start.Weekday(),
})
}
@ -177,6 +189,10 @@ loop:
chWkAdd <- cacheAdd{w: w, aid: aid, year: year, week: week, time: time.Now()}
if !includeAttsInfo && len(w.Atts) > 0 {
w.Atts = nil
}
return w, time.Time{}, nil
}

Loading…
Cancel
Save