Add infrastructure for computing and storing repository "vintages."

Will be used for caching the oldest commit that we have for a repository (for a
specific user). For now we just store the creation date if there are no commits
before that time.

app/digest.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 	"time"
 
+	"appengine"
+
 	"github.com/google/go-github/github"
 )
 
@@ -125,18 +127,18 @@ type Digest struct {
 	IntervalDigests  []*IntervalDigest
 }
 
-func newDigest(githubClient *github.Client, account *Account) (*Digest, error) {
+func newDigest(c appengine.Context, githubClient *github.Client, account *Account) (*Digest, error) {
 	user, _, err := githubClient.Users.Get("")
 	if err != nil {
 		return nil, err
 	}
 
-	repos, err := getRepos(githubClient, user)
+	repos, err := getRepos(c, githubClient, user)
 	if err != nil {
 		return nil, err
 	}
 
-	oldestDigestTime := repos.OldestFirstCommitTime.In(account.TimezoneLocation)
+	oldestDigestTime := repos.OldestVintage.In(account.TimezoneLocation)
 	intervalDigests := make([]*IntervalDigest, 0)
 	now := time.Now().In(account.TimezoneLocation)
 	for yearDelta := -1; ; yearDelta-- {

app/githop.go

@@ -204,7 +204,7 @@ func viewDigestHandler(w http.ResponseWriter, r *http.Request) {
 	oauthTransport.Token = &account.OAuthToken
 	githubClient := github.NewClient(oauthTransport.Client())
 
-	digest, err := newDigest(githubClient, account)
+	digest, err := newDigest(c, githubClient, account)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 	}
@@ -259,7 +259,7 @@ func sendDigestForAccount(account *Account, c appengine.Context) (bool, error) {
 	oauthTransport.Token = &account.OAuthToken
 	githubClient := github.NewClient(oauthTransport.Client())
 
-	digest, err := newDigest(githubClient, account)
+	digest, err := newDigest(c, githubClient, account)
 	if err != nil {
 		return false, err
 	}
@@ -356,11 +356,15 @@ func settingsHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	repos, err := getRepos(githubClient, user)
+	repos, err := getRepos(c, githubClient, user)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
 
 	var data = map[string]interface{}{
 		"Account":   account,
-		"User":			 user,
+		"User":      user,
 		"Timezones": timezones,
 		"Repos":     repos,
 	}
@@ -418,7 +422,7 @@ func digestAdminHandler(w http.ResponseWriter, r *http.Request) {
 	oauthTransport.Token = &account.OAuthToken
 	githubClient := github.NewClient(oauthTransport.Client())
 
-	digest, err := newDigest(githubClient, account)
+	digest, err := newDigest(c, githubClient, account)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 	}

app/queue.yaml

@@ -0,0 +1,4 @@
+queue:
+# Change the refresh rate of the default queue from 5/s to 50/s
+- name: default
+  rate: 50/s

app/repos.go

@@ -1,8 +1,13 @@
 package githop
 
 import (
+	"fmt"
 	"time"
 
+	"appengine"
+	"appengine/datastore"
+	"appengine/delay"
+
 	"github.com/google/go-github/github"
 )
 
@@ -10,26 +15,118 @@ const (
 	VintageDateFormat = "January 2, 2006"
 )
 
+type RepoVintage struct {
+	UserId  int       `datastore:",noindex"`
+	RepoId  int       `datastore:",noindex"`
+	Vintage time.Time `datastore:",noindex"`
+}
+
+func getVintageKey(c appengine.Context, userId int, repoId int) *datastore.Key {
+	return datastore.NewKey(c, "RepoVintage", fmt.Sprintf("%d-%d", userId, repoId), 0, nil)
+}
+
+func computeVintage(c appengine.Context, userId int, userLogin string, repoOwnerLogin string, repoName string) error {
+	account, err := getAccount(c, userId)
+	if err != nil {
+		c.Errorf("Could not load account %d: %s", userId, err.Error())
+		return err
+	}
+
+	oauthTransport := githubOAuthTransport(c)
+	oauthTransport.Token = &account.OAuthToken
+	githubClient := github.NewClient(oauthTransport.Client())
+
+	repo, _, err := githubClient.Repositories.Get(repoOwnerLogin, repoName)
+	if err != nil {
+		c.Errorf("Could not load repo %d %d: %s", repoOwnerLogin, repoName, err.Error())
+		return err
+	}
+
+	beforeCreationTime := repo.CreatedAt.UTC().AddDate(0, 0, -1)
+	commits, _, err := githubClient.Repositories.ListCommits(
+		repoOwnerLogin,
+		repoName,
+		&github.CommitsListOptions{
+			ListOptions: github.ListOptions{PerPage: 1},
+			Author:      userLogin,
+			Until:       beforeCreationTime,
+		})
+
+	if err != nil {
+		c.Errorf("Could not load commits for repo %s: %s", *repo.FullName, err.Error())
+		return err
+	}
+
+	if len(commits) > 0 {
+		// TODO: compute vintage via the stats API
+	} else {
+		_, err = datastore.Put(c, getVintageKey(c, userId, *repo.ID), &RepoVintage{
+			UserId:  userId,
+			RepoId:  *repo.ID,
+			Vintage: repo.CreatedAt.UTC(),
+		})
+		if err != nil {
+			c.Errorf("Could save vintage for repo %s: %s", *repo.FullName, err.Error())
+			return err
+		}
+	}
+
+	return nil
+}
+
+var computeVintageFunc = delay.Func("computeVintage", computeVintage)
+
+func fillVintages(c appengine.Context, user *github.User, repos []*Repo) error {
+	keys := make([]*datastore.Key, len(repos))
+	for i := range repos {
+		keys[i] = getVintageKey(c, *user.ID, *repos[i].ID)
+	}
+	vintages := make([]*RepoVintage, len(repos))
+	for i := range vintages {
+		vintages[i] = new(RepoVintage)
+	}
+	err := datastore.GetMulti(c, keys, vintages)
+	if err != nil {
+		if errs, ok := err.(appengine.MultiError); ok {
+			for i, err := range errs {
+				if err == datastore.ErrNoSuchEntity {
+					vintages[i] = nil
+				} else if err != nil {
+					c.Errorf("%d/%s vintage fetch error: %s", i, *repos[i].FullName, err.Error())
+					return err
+				}
+			}
+		} else {
+			return err
+		}
+	}
+	for i := range vintages {
+		repo := repos[i]
+		vintage := vintages[i]
+		if vintage != nil {
+			repo.Vintage = vintage.Vintage
+			continue
+		}
+		computeVintageFunc.Call(c, *user.ID, *user.Login, *repo.Owner.Login, *repo.Name)
+	}
+	return nil
+}
+
 type Repos struct {
-	AllRepos              []*Repo
+	AllRepos       []*Repo
-	UserRepos             []*Repo
+	UserRepos      []*Repo
-	OtherUserRepos        []*UserRepos
+	OtherUserRepos []*UserRepos
-	OrgRepos              []*OrgRepos
+	OrgRepos       []*OrgRepos
-	OldestFirstCommitTime time.Time
+	OldestVintage  time.Time
 }
 
 type Repo struct {
 	*github.Repository
+	Vintage time.Time
 }
 
-type UserRepos struct {
+func newRepo(githubRepo *github.Repository) *Repo {
-	User  *github.User
+	return &Repo{githubRepo, githubRepo.CreatedAt.UTC()}
-	Repos []*Repo
-}
-
-type OrgRepos struct {
-	Org   *github.Organization
-	Repos []*Repo
 }
 
 func (repo *Repo) TypeAsOcticonName() string {
@@ -53,10 +150,20 @@ func (repo *Repo) TypeAsClassName() string {
 }
 
 func (repo *Repo) DisplayVintage() string {
-	return repo.CreatedAt.Format(VintageDateFormat)
+	return repo.Vintage.Format(VintageDateFormat)
 }
 
-func getRepos(githubClient *github.Client, user *github.User) (*Repos, error) {
+type UserRepos struct {
+	User  *github.User
+	Repos []*Repo
+}
+
+type OrgRepos struct {
+	Org   *github.Organization
+	Repos []*Repo
+}
+
+func getRepos(c appengine.Context, githubClient *github.Client, user *github.User) (*Repos, error) {
 	// The username parameter must be left blank so that we can get all of the
 	// repositories the user has access to, not just ones that they own.
 	clientUserRepos, _, err := githubClient.Repositories.List("", nil)
@@ -71,7 +178,7 @@ func getRepos(githubClient *github.Client, user *github.User) (*Repos, error) {
 	for i := range clientUserRepos {
 		ownerID := *clientUserRepos[i].Owner.ID
 		if ownerID == *user.ID {
-			repos.UserRepos = append(repos.UserRepos, &Repo{&clientUserRepos[i]})
+			repos.UserRepos = append(repos.UserRepos, newRepo(&clientUserRepos[i]))
 		} else {
 			var userRepos *UserRepos
 			for j := range repos.OtherUserRepos {
@@ -87,7 +194,7 @@ func getRepos(githubClient *github.Client, user *github.User) (*Repos, error) {
 				}
 				repos.OtherUserRepos = append(repos.OtherUserRepos, userRepos)
 			}
-			userRepos.Repos = append(userRepos.Repos, &Repo{&clientUserRepos[i]})
+			userRepos.Repos = append(userRepos.Repos, newRepo(&clientUserRepos[i]))
 		}
 	}
 
@@ -107,23 +214,30 @@ func getRepos(githubClient *github.Client, user *github.User) (*Repos, error) {
 		orgRepos := make([]*Repo, 0, len(clientOrgRepos))
 		allRepoCount += len(clientOrgRepos)
 		for j := range clientOrgRepos {
-			orgRepos = append(orgRepos, &Repo{&clientOrgRepos[j]})
+			orgRepos = append(orgRepos, newRepo(&clientOrgRepos[j]))
 		}
 		repos.OrgRepos = append(repos.OrgRepos, &OrgRepos{org, orgRepos})
 	}
 
 	repos.AllRepos = make([]*Repo, 0, allRepoCount)
 	repos.AllRepos = append(repos.AllRepos, repos.UserRepos...)
+	for _, userRepos := range repos.OtherUserRepos {
+		repos.AllRepos = append(repos.AllRepos, userRepos.Repos...)
+	}
 	for _, org := range repos.OrgRepos {
 		repos.AllRepos = append(repos.AllRepos, org.Repos...)
 	}
 
-	// TODO: better computation of the oldest first commit via the stats API
+	err = fillVintages(c, user, repos.AllRepos)
-	repos.OldestFirstCommitTime = time.Now().UTC()
+	if err != nil {
+		return nil, err
+	}
+
+	repos.OldestVintage = time.Now().UTC()
 	for _, repo := range repos.AllRepos {
-		repoTime := repo.CreatedAt.UTC()
+		repoVintage := repo.Vintage
-		if repoTime.Before(repos.OldestFirstCommitTime) {
+		if repoVintage.Before(repos.OldestVintage) {
-			repos.OldestFirstCommitTime = repoTime
+			repos.OldestVintage = repoVintage
 		}
 	}