diff --git a/app/digest.go b/app/digest.go index 4cb41db..4b68219 100644 --- a/app/digest.go +++ b/app/digest.go @@ -6,6 +6,8 @@ import ( "strings" "time" + "appengine" + "github.com/google/go-github/github" ) @@ -125,18 +127,18 @@ type Digest struct { IntervalDigests []*IntervalDigest } -func newDigest(githubClient *github.Client, account *Account) (*Digest, error) { +func newDigest(c appengine.Context, githubClient *github.Client, account *Account) (*Digest, error) { user, _, err := githubClient.Users.Get("") if err != nil { return nil, err } - repos, err := getRepos(githubClient, user) + repos, err := getRepos(c, githubClient, user) if err != nil { return nil, err } - oldestDigestTime := repos.OldestFirstCommitTime.In(account.TimezoneLocation) + oldestDigestTime := repos.OldestVintage.In(account.TimezoneLocation) intervalDigests := make([]*IntervalDigest, 0) now := time.Now().In(account.TimezoneLocation) for yearDelta := -1; ; yearDelta-- { diff --git a/app/githop.go b/app/githop.go index dcf269a..2573893 100644 --- a/app/githop.go +++ b/app/githop.go @@ -204,7 +204,7 @@ func viewDigestHandler(w http.ResponseWriter, r *http.Request) { oauthTransport.Token = &account.OAuthToken githubClient := github.NewClient(oauthTransport.Client()) - digest, err := newDigest(githubClient, account) + digest, err := newDigest(c, githubClient, account) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } @@ -259,7 +259,7 @@ func sendDigestForAccount(account *Account, c appengine.Context) (bool, error) { oauthTransport.Token = &account.OAuthToken githubClient := github.NewClient(oauthTransport.Client()) - digest, err := newDigest(githubClient, account) + digest, err := newDigest(c, githubClient, account) if err != nil { return false, err } @@ -356,11 +356,15 @@ func settingsHandler(w http.ResponseWriter, r *http.Request) { return } - repos, err := getRepos(githubClient, user) + repos, err := getRepos(c, githubClient, user) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } var data = map[string]interface{}{ "Account": account, - "User": user, + "User": user, "Timezones": timezones, "Repos": repos, } @@ -418,7 +422,7 @@ func digestAdminHandler(w http.ResponseWriter, r *http.Request) { oauthTransport.Token = &account.OAuthToken githubClient := github.NewClient(oauthTransport.Client()) - digest, err := newDigest(githubClient, account) + digest, err := newDigest(c, githubClient, account) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } diff --git a/app/queue.yaml b/app/queue.yaml new file mode 100644 index 0000000..4296b1a --- /dev/null +++ b/app/queue.yaml @@ -0,0 +1,4 @@ +queue: +# Change the refresh rate of the default queue from 5/s to 50/s +- name: default + rate: 50/s diff --git a/app/repos.go b/app/repos.go index 84289a6..d8b33f2 100644 --- a/app/repos.go +++ b/app/repos.go @@ -1,8 +1,13 @@ package githop import ( + "fmt" "time" + "appengine" + "appengine/datastore" + "appengine/delay" + "github.com/google/go-github/github" ) @@ -10,26 +15,118 @@ const ( VintageDateFormat = "January 2, 2006" ) +type RepoVintage struct { + UserId int `datastore:",noindex"` + RepoId int `datastore:",noindex"` + Vintage time.Time `datastore:",noindex"` +} + +func getVintageKey(c appengine.Context, userId int, repoId int) *datastore.Key { + return datastore.NewKey(c, "RepoVintage", fmt.Sprintf("%d-%d", userId, repoId), 0, nil) +} + +func computeVintage(c appengine.Context, userId int, userLogin string, repoOwnerLogin string, repoName string) error { + account, err := getAccount(c, userId) + if err != nil { + c.Errorf("Could not load account %d: %s", userId, err.Error()) + return err + } + + oauthTransport := githubOAuthTransport(c) + oauthTransport.Token = &account.OAuthToken + githubClient := github.NewClient(oauthTransport.Client()) + + repo, _, err := githubClient.Repositories.Get(repoOwnerLogin, repoName) + if err != nil { + c.Errorf("Could not load repo %d %d: %s", repoOwnerLogin, repoName, err.Error()) + return err + } + + beforeCreationTime := repo.CreatedAt.UTC().AddDate(0, 0, -1) + commits, _, err := githubClient.Repositories.ListCommits( + repoOwnerLogin, + repoName, + &github.CommitsListOptions{ + ListOptions: github.ListOptions{PerPage: 1}, + Author: userLogin, + Until: beforeCreationTime, + }) + + if err != nil { + c.Errorf("Could not load commits for repo %s: %s", *repo.FullName, err.Error()) + return err + } + + if len(commits) > 0 { + // TODO: compute vintage via the stats API + } else { + _, err = datastore.Put(c, getVintageKey(c, userId, *repo.ID), &RepoVintage{ + UserId: userId, + RepoId: *repo.ID, + Vintage: repo.CreatedAt.UTC(), + }) + if err != nil { + c.Errorf("Could save vintage for repo %s: %s", *repo.FullName, err.Error()) + return err + } + } + + return nil +} + +var computeVintageFunc = delay.Func("computeVintage", computeVintage) + +func fillVintages(c appengine.Context, user *github.User, repos []*Repo) error { + keys := make([]*datastore.Key, len(repos)) + for i := range repos { + keys[i] = getVintageKey(c, *user.ID, *repos[i].ID) + } + vintages := make([]*RepoVintage, len(repos)) + for i := range vintages { + vintages[i] = new(RepoVintage) + } + err := datastore.GetMulti(c, keys, vintages) + if err != nil { + if errs, ok := err.(appengine.MultiError); ok { + for i, err := range errs { + if err == datastore.ErrNoSuchEntity { + vintages[i] = nil + } else if err != nil { + c.Errorf("%d/%s vintage fetch error: %s", i, *repos[i].FullName, err.Error()) + return err + } + } + } else { + return err + } + } + for i := range vintages { + repo := repos[i] + vintage := vintages[i] + if vintage != nil { + repo.Vintage = vintage.Vintage + continue + } + computeVintageFunc.Call(c, *user.ID, *user.Login, *repo.Owner.Login, *repo.Name) + } + return nil +} + type Repos struct { - AllRepos []*Repo - UserRepos []*Repo - OtherUserRepos []*UserRepos - OrgRepos []*OrgRepos - OldestFirstCommitTime time.Time + AllRepos []*Repo + UserRepos []*Repo + OtherUserRepos []*UserRepos + OrgRepos []*OrgRepos + OldestVintage time.Time } type Repo struct { *github.Repository + Vintage time.Time } -type UserRepos struct { - User *github.User - Repos []*Repo -} - -type OrgRepos struct { - Org *github.Organization - Repos []*Repo +func newRepo(githubRepo *github.Repository) *Repo { + return &Repo{githubRepo, githubRepo.CreatedAt.UTC()} } func (repo *Repo) TypeAsOcticonName() string { @@ -53,10 +150,20 @@ func (repo *Repo) TypeAsClassName() string { } func (repo *Repo) DisplayVintage() string { - return repo.CreatedAt.Format(VintageDateFormat) + return repo.Vintage.Format(VintageDateFormat) } -func getRepos(githubClient *github.Client, user *github.User) (*Repos, error) { +type UserRepos struct { + User *github.User + Repos []*Repo +} + +type OrgRepos struct { + Org *github.Organization + Repos []*Repo +} + +func getRepos(c appengine.Context, githubClient *github.Client, user *github.User) (*Repos, error) { // The username parameter must be left blank so that we can get all of the // repositories the user has access to, not just ones that they own. clientUserRepos, _, err := githubClient.Repositories.List("", nil) @@ -71,7 +178,7 @@ func getRepos(githubClient *github.Client, user *github.User) (*Repos, error) { for i := range clientUserRepos { ownerID := *clientUserRepos[i].Owner.ID if ownerID == *user.ID { - repos.UserRepos = append(repos.UserRepos, &Repo{&clientUserRepos[i]}) + repos.UserRepos = append(repos.UserRepos, newRepo(&clientUserRepos[i])) } else { var userRepos *UserRepos for j := range repos.OtherUserRepos { @@ -87,7 +194,7 @@ func getRepos(githubClient *github.Client, user *github.User) (*Repos, error) { } repos.OtherUserRepos = append(repos.OtherUserRepos, userRepos) } - userRepos.Repos = append(userRepos.Repos, &Repo{&clientUserRepos[i]}) + userRepos.Repos = append(userRepos.Repos, newRepo(&clientUserRepos[i])) } } @@ -107,23 +214,30 @@ func getRepos(githubClient *github.Client, user *github.User) (*Repos, error) { orgRepos := make([]*Repo, 0, len(clientOrgRepos)) allRepoCount += len(clientOrgRepos) for j := range clientOrgRepos { - orgRepos = append(orgRepos, &Repo{&clientOrgRepos[j]}) + orgRepos = append(orgRepos, newRepo(&clientOrgRepos[j])) } repos.OrgRepos = append(repos.OrgRepos, &OrgRepos{org, orgRepos}) } repos.AllRepos = make([]*Repo, 0, allRepoCount) repos.AllRepos = append(repos.AllRepos, repos.UserRepos...) + for _, userRepos := range repos.OtherUserRepos { + repos.AllRepos = append(repos.AllRepos, userRepos.Repos...) + } for _, org := range repos.OrgRepos { repos.AllRepos = append(repos.AllRepos, org.Repos...) } - // TODO: better computation of the oldest first commit via the stats API - repos.OldestFirstCommitTime = time.Now().UTC() + err = fillVintages(c, user, repos.AllRepos) + if err != nil { + return nil, err + } + + repos.OldestVintage = time.Now().UTC() for _, repo := range repos.AllRepos { - repoTime := repo.CreatedAt.UTC() - if repoTime.Before(repos.OldestFirstCommitTime) { - repos.OldestFirstCommitTime = repoTime + repoVintage := repo.Vintage + if repoVintage.Before(repos.OldestVintage) { + repos.OldestVintage = repoVintage } }