Small refactor

.gitignore

@@ -9,3 +9,4 @@ docSet.dsidx
 docs.min.css
 holygrail.min.css
 syntax-highlighter.min.css
+*-version.txt

SFDashC/database.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"database/sql"
+	"github.com/coopernurse/gorp"
+	_ "github.com/mattn/go-sqlite3"
+)
+
+var dbmap *gorp.DbMap
+
+func InitDb() *gorp.DbMap {
+	db, err := sql.Open("sqlite3", "docSet.dsidx")
+	ExitIfError(err)
+
+	dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}
+
+	dbmap.AddTableWithName(SearchIndex{}, "searchIndex").SetKeys(true, "ID")
+
+	err = dbmap.CreateTablesIfNotExists()
+	ExitIfError(err)
+
+	err = dbmap.TruncateTables()
+	ExitIfError(err)
+
+	return dbmap
+}
+
+func SaveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
+	if entry.LinkAttr.Href == "" || entryType == nil {
+		return
+	}
+
+	relLink := entry.GetContentFilepath(toc, false)
+	name := entry.CleanTitle(*entryType)
+	if entryType.ShowNamespace && len(entryHierarchy) > 0 {
+		// Show namespace for methods
+		name = entryHierarchy[len(entryHierarchy)-1] + "." + name
+	}
+
+	// fmt.Println("Storing: " + name)
+
+	si := SearchIndex{
+		Name: name,
+		Type: entryType.TypeName,
+		Path: relLink,
+	}
+
+	dbmap.Insert(&si)
+}

SFDashC/main.go

@@ -1,12 +1,9 @@
 package main
 
 import (
-	"database/sql"
 	"encoding/json"
 	"flag"
 	"fmt"
-	"github.com/coopernurse/gorp"
-	_ "github.com/mattn/go-sqlite3"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -16,88 +13,15 @@ import (
 	"sync"
 )
 
-/*
-TODO:
- - Move structs to own file
- - Move db stuff to own file
- - Stylesheets
-*/
-
 // CSS Paths
 var cssBasePath = "https://developer.salesforce.com/resource/stylesheets"
 var cssFiles = []string{"holygrail.min.css", "docs.min.css", "syntax-highlighter.min.css"}
 
-// JSON Structs
-
-// AtlasTOC represents the meta documenation from Salesforce
-type AtlasTOC struct {
-	AvailableVersions []VersionInfo `json:"available_versions"`
-	Content           string
-	ContentDocumentID string `json:"content_document_id"`
-	Deliverable       string
-	DocTitle          string `json:"doc_title"`
-	Locale            string
-	Language          LanguageInfo
-	PDFUrl            string     `json:"pdf_url"`
-	TOCEntries        []TOCEntry `json:"toc"`
-	Title             string
-	Version           VersionInfo
-}
-
-// LanguageInfo contains information for linking and displaying the language
-type LanguageInfo struct {
-	Label  string
-	Locale string
-	URL    string
-}
-
-// VersionInfo representes a Salesforce documentation version
-type VersionInfo struct {
-	DocVersion     string `json:"doc_version"`
-	ReleaseVersion string `json:"release_version"`
-	VersionText    string `json:"version_text"`
-	VersionURL     string `json:"version_url"`
-}
-
-// TOCEntry represents a single Table of Contents item
-type TOCEntry struct {
-	Text                    string
-	ID                      string
-	LinkAttr                LinkAttr `json:"a_attr,omitempty"`
-	Children                []TOCEntry
-	ComputedFirstTopic      bool
-	ComputedResetPageLayout bool
-}
-
-// LinkAttr represents all attributes bound to a link
-type LinkAttr struct {
-	Href string
-}
-
-// TOCContent contains content information for a piece of documenation
-type TOCContent struct {
-	ID      string
-	Title   string
-	Content string
-}
-
-// Sqlite Struct
-
-// SearchIndex is the database table that indexes the docs
-type SearchIndex struct {
-	ID   int64  `db:id`
-	Name string `db:name`
-	Type string `db:type`
-	Path string `db:path`
-}
-
-var dbmap *gorp.DbMap
 var wg sync.WaitGroup
+var throttle = make(chan int, maxConcurrency)
 
 const maxConcurrency = 16
 
-var throttle = make(chan int, maxConcurrency)
-
 func parseFlags() (locale string, deliverables []string, silent bool) {
 	flag.StringVar(
 		&locale, "locale", "en-us",
@@ -172,6 +96,19 @@ func saveMainContent(toc *AtlasTOC) {
 	}
 }
 
+func saveContentVersion(toc *AtlasTOC) {
+	filePath := fmt.Sprintf("%s-version.txt", toc.Deliverable)
+	err := os.MkdirAll(filepath.Dir(filePath), 0755)
+	ExitIfError(err)
+
+	ofile, err := os.Create(filePath)
+	ExitIfError(err)
+
+	defer ofile.Close()
+	_, err = ofile.WriteString(toc.Version.DocVersion)
+	ExitIfError(err)
+}
+
 func main() {
 	locale, deliverables, silent := parseFlags()
 	if silent {
@@ -186,19 +123,19 @@ func main() {
 	}
 
 	// Init the Sqlite db
-	dbmap = initDb()
+	dbmap = InitDb()
 	err := dbmap.TruncateTables()
 	ExitIfError(err)
 
 	for _, deliverable := range deliverables {
 		toc, err := getTOC(locale, deliverable)
-		ExitIfError(err)
-
-		saveMainContent(toc)
 
 		err = verifyVersion(toc)
 		WarnIfError(err)
 
+		saveMainContent(toc)
+		saveContentVersion(toc)
+
 		// Download each entry
 		for _, entry := range toc.TOCEntries {
 			if entry.ID == "apex_reference" || entry.ID == "pages_compref" {
@@ -319,78 +256,6 @@ var supportedTypes = []SupportedType{
 	},
 }
 
-// IsType indicates that the TOCEntry is of a given SupportedType
-// This is done by checking the suffix of the entry text
-func (entry TOCEntry) IsType(t SupportedType) bool {
-	return strings.HasSuffix(entry.Text, t.TitleSuffix)
-}
-
-// CleanTitle trims known suffix from TOCEntry titles
-func (entry TOCEntry) CleanTitle(t SupportedType) string {
-	if t.NoTrim {
-		return entry.Text
-	}
-	return strings.TrimSuffix(entry.Text, " "+t.TitleSuffix)
-}
-
-// GetRelLink extracts only the relative link from the Link Href
-func (entry TOCEntry) GetRelLink(removeAnchor bool) (relLink string) {
-	if entry.LinkAttr.Href == "" {
-		return
-	}
-
-	// Get the JSON file
-	relLink = entry.LinkAttr.Href
-	if removeAnchor {
-		anchorIndex := strings.LastIndex(relLink, "#")
-		if anchorIndex > 0 {
-			relLink = relLink[0:anchorIndex]
-		}
-	}
-	return
-}
-
-// GetContent retrieves Content for this TOCEntry from the API
-func (entry TOCEntry) GetContent(toc *AtlasTOC) (content *TOCContent, err error) {
-	relLink := entry.GetRelLink(true)
-	if relLink == "" {
-		return
-	}
-
-	url := fmt.Sprintf(
-		"https://developer.salesforce.com/docs/get_document_content/%s/%s/%s/%s",
-		toc.Deliverable,
-		relLink,
-		toc.Locale,
-		toc.Version.DocVersion,
-	)
-
-	// fmt.Println(url)
-	resp, err := http.Get(url)
-	if err != nil {
-		return
-	}
-
-	// Read the downloaded JSON
-	defer resp.Body.Close()
-	contents, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return
-	}
-
-	// Load into Struct
-	content = new(TOCContent)
-	err = json.Unmarshal([]byte(contents), content)
-	if err != nil {
-		fmt.Println("Error reading JSON")
-		fmt.Println(resp.Status)
-		fmt.Println(url)
-		fmt.Println(string(contents))
-		return
-	}
-	return
-}
-
 func getEntryType(entry TOCEntry) (*SupportedType, error) {
 	if strings.HasPrefix(entry.ID, "pages_compref_") {
 		return &SupportedType{
@@ -425,9 +290,9 @@ func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *Atlas
 
 			childType, err = getEntryType(child)
 			if childType == nil && (entryType != nil && entryType.IsContainer) {
-				saveSearchIndex(dbmap, child, entryType, toc)
+				SaveSearchIndex(dbmap, child, entryType, toc)
 			} else if childType != nil && !childType.IsContainer {
-				saveSearchIndex(dbmap, child, childType, toc)
+				SaveSearchIndex(dbmap, child, childType, toc)
 			} else {
 				WarnIfError(err)
 			}
@@ -443,16 +308,6 @@ func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *Atlas
 	}
 }
 
-// GetContentFilepath returns the filepath that should be used for the content
-func (entry TOCEntry) GetContentFilepath(toc *AtlasTOC, removeAnchor bool) string {
-	relLink := entry.GetRelLink(removeAnchor)
-	if relLink == "" {
-		ExitIfError(NewFormatedError("Link not found for %s", entry.ID))
-	}
-
-	return fmt.Sprintf("atlas.%s.%s.meta/%s/%s", toc.Locale, toc.Deliverable, toc.Deliverable, relLink)
-}
-
 func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) {
 	defer wg.Done()
 
@@ -508,47 +363,3 @@ func downloadCSS(fileName string, wg *sync.WaitGroup) {
 
 	<-throttle
 }
-
-/**********************
-    Database
-**********************/
-
-func saveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
-	if entry.LinkAttr.Href == "" || entryType == nil {
-		return
-	}
-
-	relLink := entry.GetContentFilepath(toc, false)
-	name := entry.CleanTitle(*entryType)
-	if entryType.ShowNamespace && len(entryHierarchy) > 0 {
-		// Show namespace for methods
-		name = entryHierarchy[len(entryHierarchy)-1] + "." + name
-	}
-
-	// fmt.Println("Storing: " + name)
-
-	si := SearchIndex{
-		Name: name,
-		Type: entryType.TypeName,
-		Path: relLink,
-	}
-
-	dbmap.Insert(&si)
-}
-
-func initDb() *gorp.DbMap {
-	db, err := sql.Open("sqlite3", "docSet.dsidx")
-	ExitIfError(err)
-
-	dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}
-
-	dbmap.AddTableWithName(SearchIndex{}, "searchIndex").SetKeys(true, "ID")
-
-	err = dbmap.CreateTablesIfNotExists()
-	ExitIfError(err)
-
-	err = dbmap.TruncateTables()
-	ExitIfError(err)
-
-	return dbmap
-}

SFDashC/structs.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// JSON Structs
+
+// AtlasTOC represents the meta documenation from Salesforce
+type AtlasTOC struct {
+	AvailableVersions []VersionInfo `json:"available_versions"`
+	Content           string
+	ContentDocumentID string `json:"content_document_id"`
+	Deliverable       string
+	DocTitle          string `json:"doc_title"`
+	Locale            string
+	Language          LanguageInfo
+	PDFUrl            string     `json:"pdf_url"`
+	TOCEntries        []TOCEntry `json:"toc"`
+	Title             string
+	Version           VersionInfo
+}
+
+// LanguageInfo contains information for linking and displaying the language
+type LanguageInfo struct {
+	Label  string
+	Locale string
+	URL    string
+}
+
+// VersionInfo representes a Salesforce documentation version
+type VersionInfo struct {
+	DocVersion     string `json:"doc_version"`
+	ReleaseVersion string `json:"release_version"`
+	VersionText    string `json:"version_text"`
+	VersionURL     string `json:"version_url"`
+}
+
+// TOCEntry represents a single Table of Contents item
+type TOCEntry struct {
+	Text                    string
+	ID                      string
+	LinkAttr                LinkAttr `json:"a_attr,omitempty"`
+	Children                []TOCEntry
+	ComputedFirstTopic      bool
+	ComputedResetPageLayout bool
+}
+
+// LinkAttr represents all attributes bound to a link
+type LinkAttr struct {
+	Href string
+}
+
+// TOCContent contains content information for a piece of documenation
+type TOCContent struct {
+	ID      string
+	Title   string
+	Content string
+}
+
+// Sqlite Struct
+
+// SearchIndex is the database table that indexes the docs
+type SearchIndex struct {
+	ID   int64  `db:id`
+	Name string `db:name`
+	Type string `db:type`
+	Path string `db:path`
+}
+
+// IsType indicates that the TOCEntry is of a given SupportedType
+// This is done by checking the suffix of the entry text
+func (entry TOCEntry) IsType(t SupportedType) bool {
+	return strings.HasSuffix(entry.Text, t.TitleSuffix)
+}
+
+// CleanTitle trims known suffix from TOCEntry titles
+func (entry TOCEntry) CleanTitle(t SupportedType) string {
+	if t.NoTrim {
+		return entry.Text
+	}
+	return strings.TrimSuffix(entry.Text, " "+t.TitleSuffix)
+}
+
+// GetRelLink extracts only the relative link from the Link Href
+func (entry TOCEntry) GetRelLink(removeAnchor bool) (relLink string) {
+	if entry.LinkAttr.Href == "" {
+		return
+	}
+
+	// Get the JSON file
+	relLink = entry.LinkAttr.Href
+	if removeAnchor {
+		anchorIndex := strings.LastIndex(relLink, "#")
+		if anchorIndex > 0 {
+			relLink = relLink[0:anchorIndex]
+		}
+	}
+	return
+}
+
+// GetContent retrieves Content for this TOCEntry from the API
+func (entry TOCEntry) GetContent(toc *AtlasTOC) (content *TOCContent, err error) {
+	relLink := entry.GetRelLink(true)
+	if relLink == "" {
+		return
+	}
+
+	url := fmt.Sprintf(
+		"https://developer.salesforce.com/docs/get_document_content/%s/%s/%s/%s",
+		toc.Deliverable,
+		relLink,
+		toc.Locale,
+		toc.Version.DocVersion,
+	)
+
+	// fmt.Println(url)
+	resp, err := http.Get(url)
+	if err != nil {
+		return
+	}
+
+	// Read the downloaded JSON
+	defer resp.Body.Close()
+	contents, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+
+	// Load into Struct
+	content = new(TOCContent)
+	err = json.Unmarshal([]byte(contents), content)
+	if err != nil {
+		fmt.Println("Error reading JSON")
+		fmt.Println(resp.Status)
+		fmt.Println(url)
+		fmt.Println(string(contents))
+		return
+	}
+	return
+}
+
+// GetContentFilepath returns the filepath that should be used for the content
+func (entry TOCEntry) GetContentFilepath(toc *AtlasTOC, removeAnchor bool) string {
+	relLink := entry.GetRelLink(removeAnchor)
+	if relLink == "" {
+		ExitIfError(NewFormatedError("Link not found for %s", entry.ID))
+	}
+
+	return fmt.Sprintf("atlas.%s.%s.meta/%s/%s", toc.Locale, toc.Deliverable, toc.Deliverable, relLink)
+}