diff --git a/.gitignore b/.gitignore index 798fe87..c7263c0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ docSet.dsidx docs.min.css holygrail.min.css syntax-highlighter.min.css +*-version.txt diff --git a/SFDashC/database.go b/SFDashC/database.go new file mode 100644 index 0000000..e917215 --- /dev/null +++ b/SFDashC/database.go @@ -0,0 +1,49 @@ +package main + +import ( + "database/sql" + "github.com/coopernurse/gorp" + _ "github.com/mattn/go-sqlite3" +) + +var dbmap *gorp.DbMap + +func InitDb() *gorp.DbMap { + db, err := sql.Open("sqlite3", "docSet.dsidx") + ExitIfError(err) + + dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}} + + dbmap.AddTableWithName(SearchIndex{}, "searchIndex").SetKeys(true, "ID") + + err = dbmap.CreateTablesIfNotExists() + ExitIfError(err) + + err = dbmap.TruncateTables() + ExitIfError(err) + + return dbmap +} + +func SaveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) { + if entry.LinkAttr.Href == "" || entryType == nil { + return + } + + relLink := entry.GetContentFilepath(toc, false) + name := entry.CleanTitle(*entryType) + if entryType.ShowNamespace && len(entryHierarchy) > 0 { + // Show namespace for methods + name = entryHierarchy[len(entryHierarchy)-1] + "." + name + } + + // fmt.Println("Storing: " + name) + + si := SearchIndex{ + Name: name, + Type: entryType.TypeName, + Path: relLink, + } + + dbmap.Insert(&si) +} diff --git a/SFDashC/main.go b/SFDashC/main.go index 09aac6b..01e91d2 100644 --- a/SFDashC/main.go +++ b/SFDashC/main.go @@ -1,12 +1,9 @@ package main import ( - "database/sql" "encoding/json" "flag" "fmt" - "github.com/coopernurse/gorp" - _ "github.com/mattn/go-sqlite3" "io" "io/ioutil" "net/http" @@ -16,88 +13,15 @@ import ( "sync" ) -/* -TODO: - - Move structs to own file - - Move db stuff to own file - - Stylesheets -*/ - // CSS Paths var cssBasePath = "https://developer.salesforce.com/resource/stylesheets" var cssFiles = []string{"holygrail.min.css", "docs.min.css", "syntax-highlighter.min.css"} -// JSON Structs - -// AtlasTOC represents the meta documenation from Salesforce -type AtlasTOC struct { - AvailableVersions []VersionInfo `json:"available_versions"` - Content string - ContentDocumentID string `json:"content_document_id"` - Deliverable string - DocTitle string `json:"doc_title"` - Locale string - Language LanguageInfo - PDFUrl string `json:"pdf_url"` - TOCEntries []TOCEntry `json:"toc"` - Title string - Version VersionInfo -} - -// LanguageInfo contains information for linking and displaying the language -type LanguageInfo struct { - Label string - Locale string - URL string -} - -// VersionInfo representes a Salesforce documentation version -type VersionInfo struct { - DocVersion string `json:"doc_version"` - ReleaseVersion string `json:"release_version"` - VersionText string `json:"version_text"` - VersionURL string `json:"version_url"` -} - -// TOCEntry represents a single Table of Contents item -type TOCEntry struct { - Text string - ID string - LinkAttr LinkAttr `json:"a_attr,omitempty"` - Children []TOCEntry - ComputedFirstTopic bool - ComputedResetPageLayout bool -} - -// LinkAttr represents all attributes bound to a link -type LinkAttr struct { - Href string -} - -// TOCContent contains content information for a piece of documenation -type TOCContent struct { - ID string - Title string - Content string -} - -// Sqlite Struct - -// SearchIndex is the database table that indexes the docs -type SearchIndex struct { - ID int64 `db:id` - Name string `db:name` - Type string `db:type` - Path string `db:path` -} - -var dbmap *gorp.DbMap var wg sync.WaitGroup +var throttle = make(chan int, maxConcurrency) const maxConcurrency = 16 -var throttle = make(chan int, maxConcurrency) - func parseFlags() (locale string, deliverables []string, silent bool) { flag.StringVar( &locale, "locale", "en-us", @@ -172,6 +96,19 @@ func saveMainContent(toc *AtlasTOC) { } } +func saveContentVersion(toc *AtlasTOC) { + filePath := fmt.Sprintf("%s-version.txt", toc.Deliverable) + err := os.MkdirAll(filepath.Dir(filePath), 0755) + ExitIfError(err) + + ofile, err := os.Create(filePath) + ExitIfError(err) + + defer ofile.Close() + _, err = ofile.WriteString(toc.Version.DocVersion) + ExitIfError(err) +} + func main() { locale, deliverables, silent := parseFlags() if silent { @@ -186,19 +123,19 @@ func main() { } // Init the Sqlite db - dbmap = initDb() + dbmap = InitDb() err := dbmap.TruncateTables() ExitIfError(err) for _, deliverable := range deliverables { toc, err := getTOC(locale, deliverable) - ExitIfError(err) - - saveMainContent(toc) err = verifyVersion(toc) WarnIfError(err) + saveMainContent(toc) + saveContentVersion(toc) + // Download each entry for _, entry := range toc.TOCEntries { if entry.ID == "apex_reference" || entry.ID == "pages_compref" { @@ -319,78 +256,6 @@ var supportedTypes = []SupportedType{ }, } -// IsType indicates that the TOCEntry is of a given SupportedType -// This is done by checking the suffix of the entry text -func (entry TOCEntry) IsType(t SupportedType) bool { - return strings.HasSuffix(entry.Text, t.TitleSuffix) -} - -// CleanTitle trims known suffix from TOCEntry titles -func (entry TOCEntry) CleanTitle(t SupportedType) string { - if t.NoTrim { - return entry.Text - } - return strings.TrimSuffix(entry.Text, " "+t.TitleSuffix) -} - -// GetRelLink extracts only the relative link from the Link Href -func (entry TOCEntry) GetRelLink(removeAnchor bool) (relLink string) { - if entry.LinkAttr.Href == "" { - return - } - - // Get the JSON file - relLink = entry.LinkAttr.Href - if removeAnchor { - anchorIndex := strings.LastIndex(relLink, "#") - if anchorIndex > 0 { - relLink = relLink[0:anchorIndex] - } - } - return -} - -// GetContent retrieves Content for this TOCEntry from the API -func (entry TOCEntry) GetContent(toc *AtlasTOC) (content *TOCContent, err error) { - relLink := entry.GetRelLink(true) - if relLink == "" { - return - } - - url := fmt.Sprintf( - "https://developer.salesforce.com/docs/get_document_content/%s/%s/%s/%s", - toc.Deliverable, - relLink, - toc.Locale, - toc.Version.DocVersion, - ) - - // fmt.Println(url) - resp, err := http.Get(url) - if err != nil { - return - } - - // Read the downloaded JSON - defer resp.Body.Close() - contents, err := ioutil.ReadAll(resp.Body) - if err != nil { - return - } - - // Load into Struct - content = new(TOCContent) - err = json.Unmarshal([]byte(contents), content) - if err != nil { - fmt.Println("Error reading JSON") - fmt.Println(resp.Status) - fmt.Println(url) - fmt.Println(string(contents)) - return - } - return -} - func getEntryType(entry TOCEntry) (*SupportedType, error) { if strings.HasPrefix(entry.ID, "pages_compref_") { return &SupportedType{ @@ -425,9 +290,9 @@ func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *Atlas childType, err = getEntryType(child) if childType == nil && (entryType != nil && entryType.IsContainer) { - saveSearchIndex(dbmap, child, entryType, toc) + SaveSearchIndex(dbmap, child, entryType, toc) } else if childType != nil && !childType.IsContainer { - saveSearchIndex(dbmap, child, childType, toc) + SaveSearchIndex(dbmap, child, childType, toc) } else { WarnIfError(err) } @@ -443,16 +308,6 @@ func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *Atlas } } -// GetContentFilepath returns the filepath that should be used for the content -func (entry TOCEntry) GetContentFilepath(toc *AtlasTOC, removeAnchor bool) string { - relLink := entry.GetRelLink(removeAnchor) - if relLink == "" { - ExitIfError(NewFormatedError("Link not found for %s", entry.ID)) - } - - return fmt.Sprintf("atlas.%s.%s.meta/%s/%s", toc.Locale, toc.Deliverable, toc.Deliverable, relLink) -} - func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) { defer wg.Done() @@ -508,47 +363,3 @@ func downloadCSS(fileName string, wg *sync.WaitGroup) { <-throttle } - -/********************** - Database -**********************/ - -func saveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) { - if entry.LinkAttr.Href == "" || entryType == nil { - return - } - - relLink := entry.GetContentFilepath(toc, false) - name := entry.CleanTitle(*entryType) - if entryType.ShowNamespace && len(entryHierarchy) > 0 { - // Show namespace for methods - name = entryHierarchy[len(entryHierarchy)-1] + "." + name - } - - // fmt.Println("Storing: " + name) - - si := SearchIndex{ - Name: name, - Type: entryType.TypeName, - Path: relLink, - } - - dbmap.Insert(&si) -} - -func initDb() *gorp.DbMap { - db, err := sql.Open("sqlite3", "docSet.dsidx") - ExitIfError(err) - - dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}} - - dbmap.AddTableWithName(SearchIndex{}, "searchIndex").SetKeys(true, "ID") - - err = dbmap.CreateTablesIfNotExists() - ExitIfError(err) - - err = dbmap.TruncateTables() - ExitIfError(err) - - return dbmap -} diff --git a/SFDashC/structs.go b/SFDashC/structs.go new file mode 100644 index 0000000..65bb72b --- /dev/null +++ b/SFDashC/structs.go @@ -0,0 +1,155 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" +) + +// JSON Structs + +// AtlasTOC represents the meta documenation from Salesforce +type AtlasTOC struct { + AvailableVersions []VersionInfo `json:"available_versions"` + Content string + ContentDocumentID string `json:"content_document_id"` + Deliverable string + DocTitle string `json:"doc_title"` + Locale string + Language LanguageInfo + PDFUrl string `json:"pdf_url"` + TOCEntries []TOCEntry `json:"toc"` + Title string + Version VersionInfo +} + +// LanguageInfo contains information for linking and displaying the language +type LanguageInfo struct { + Label string + Locale string + URL string +} + +// VersionInfo representes a Salesforce documentation version +type VersionInfo struct { + DocVersion string `json:"doc_version"` + ReleaseVersion string `json:"release_version"` + VersionText string `json:"version_text"` + VersionURL string `json:"version_url"` +} + +// TOCEntry represents a single Table of Contents item +type TOCEntry struct { + Text string + ID string + LinkAttr LinkAttr `json:"a_attr,omitempty"` + Children []TOCEntry + ComputedFirstTopic bool + ComputedResetPageLayout bool +} + +// LinkAttr represents all attributes bound to a link +type LinkAttr struct { + Href string +} + +// TOCContent contains content information for a piece of documenation +type TOCContent struct { + ID string + Title string + Content string +} + +// Sqlite Struct + +// SearchIndex is the database table that indexes the docs +type SearchIndex struct { + ID int64 `db:id` + Name string `db:name` + Type string `db:type` + Path string `db:path` +} + +// IsType indicates that the TOCEntry is of a given SupportedType +// This is done by checking the suffix of the entry text +func (entry TOCEntry) IsType(t SupportedType) bool { + return strings.HasSuffix(entry.Text, t.TitleSuffix) +} + +// CleanTitle trims known suffix from TOCEntry titles +func (entry TOCEntry) CleanTitle(t SupportedType) string { + if t.NoTrim { + return entry.Text + } + return strings.TrimSuffix(entry.Text, " "+t.TitleSuffix) +} + +// GetRelLink extracts only the relative link from the Link Href +func (entry TOCEntry) GetRelLink(removeAnchor bool) (relLink string) { + if entry.LinkAttr.Href == "" { + return + } + + // Get the JSON file + relLink = entry.LinkAttr.Href + if removeAnchor { + anchorIndex := strings.LastIndex(relLink, "#") + if anchorIndex > 0 { + relLink = relLink[0:anchorIndex] + } + } + return +} + +// GetContent retrieves Content for this TOCEntry from the API +func (entry TOCEntry) GetContent(toc *AtlasTOC) (content *TOCContent, err error) { + relLink := entry.GetRelLink(true) + if relLink == "" { + return + } + + url := fmt.Sprintf( + "https://developer.salesforce.com/docs/get_document_content/%s/%s/%s/%s", + toc.Deliverable, + relLink, + toc.Locale, + toc.Version.DocVersion, + ) + + // fmt.Println(url) + resp, err := http.Get(url) + if err != nil { + return + } + + // Read the downloaded JSON + defer resp.Body.Close() + contents, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + // Load into Struct + content = new(TOCContent) + err = json.Unmarshal([]byte(contents), content) + if err != nil { + fmt.Println("Error reading JSON") + fmt.Println(resp.Status) + fmt.Println(url) + fmt.Println(string(contents)) + return + } + return +} + +// GetContentFilepath returns the filepath that should be used for the content +func (entry TOCEntry) GetContentFilepath(toc *AtlasTOC, removeAnchor bool) string { + relLink := entry.GetRelLink(removeAnchor) + if relLink == "" { + ExitIfError(NewFormatedError("Link not found for %s", entry.ID)) + } + + return fmt.Sprintf("atlas.%s.%s.meta/%s/%s", toc.Locale, toc.Deliverable, toc.Deliverable, relLink) +}