Small refactor

This commit is contained in:
ViViDboarder 2016-07-26 18:26:47 -07:00
parent e64ac5df22
commit 80d6c867cd
4 changed files with 225 additions and 209 deletions

1
.gitignore vendored
View File

@ -9,3 +9,4 @@ docSet.dsidx
docs.min.css docs.min.css
holygrail.min.css holygrail.min.css
syntax-highlighter.min.css syntax-highlighter.min.css
*-version.txt

49
SFDashC/database.go Normal file
View File

@ -0,0 +1,49 @@
package main
import (
"database/sql"
"github.com/coopernurse/gorp"
_ "github.com/mattn/go-sqlite3"
)
var dbmap *gorp.DbMap
func InitDb() *gorp.DbMap {
db, err := sql.Open("sqlite3", "docSet.dsidx")
ExitIfError(err)
dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}
dbmap.AddTableWithName(SearchIndex{}, "searchIndex").SetKeys(true, "ID")
err = dbmap.CreateTablesIfNotExists()
ExitIfError(err)
err = dbmap.TruncateTables()
ExitIfError(err)
return dbmap
}
func SaveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
if entry.LinkAttr.Href == "" || entryType == nil {
return
}
relLink := entry.GetContentFilepath(toc, false)
name := entry.CleanTitle(*entryType)
if entryType.ShowNamespace && len(entryHierarchy) > 0 {
// Show namespace for methods
name = entryHierarchy[len(entryHierarchy)-1] + "." + name
}
// fmt.Println("Storing: " + name)
si := SearchIndex{
Name: name,
Type: entryType.TypeName,
Path: relLink,
}
dbmap.Insert(&si)
}

View File

@ -1,12 +1,9 @@
package main package main
import ( import (
"database/sql"
"encoding/json" "encoding/json"
"flag" "flag"
"fmt" "fmt"
"github.com/coopernurse/gorp"
_ "github.com/mattn/go-sqlite3"
"io" "io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@ -16,88 +13,15 @@ import (
"sync" "sync"
) )
/*
TODO:
- Move structs to own file
- Move db stuff to own file
- Stylesheets
*/
// CSS Paths // CSS Paths
var cssBasePath = "https://developer.salesforce.com/resource/stylesheets" var cssBasePath = "https://developer.salesforce.com/resource/stylesheets"
var cssFiles = []string{"holygrail.min.css", "docs.min.css", "syntax-highlighter.min.css"} var cssFiles = []string{"holygrail.min.css", "docs.min.css", "syntax-highlighter.min.css"}
// JSON Structs
// AtlasTOC represents the meta documenation from Salesforce
type AtlasTOC struct {
AvailableVersions []VersionInfo `json:"available_versions"`
Content string
ContentDocumentID string `json:"content_document_id"`
Deliverable string
DocTitle string `json:"doc_title"`
Locale string
Language LanguageInfo
PDFUrl string `json:"pdf_url"`
TOCEntries []TOCEntry `json:"toc"`
Title string
Version VersionInfo
}
// LanguageInfo contains information for linking and displaying the language
type LanguageInfo struct {
Label string
Locale string
URL string
}
// VersionInfo representes a Salesforce documentation version
type VersionInfo struct {
DocVersion string `json:"doc_version"`
ReleaseVersion string `json:"release_version"`
VersionText string `json:"version_text"`
VersionURL string `json:"version_url"`
}
// TOCEntry represents a single Table of Contents item
type TOCEntry struct {
Text string
ID string
LinkAttr LinkAttr `json:"a_attr,omitempty"`
Children []TOCEntry
ComputedFirstTopic bool
ComputedResetPageLayout bool
}
// LinkAttr represents all attributes bound to a link
type LinkAttr struct {
Href string
}
// TOCContent contains content information for a piece of documenation
type TOCContent struct {
ID string
Title string
Content string
}
// Sqlite Struct
// SearchIndex is the database table that indexes the docs
type SearchIndex struct {
ID int64 `db:id`
Name string `db:name`
Type string `db:type`
Path string `db:path`
}
var dbmap *gorp.DbMap
var wg sync.WaitGroup var wg sync.WaitGroup
var throttle = make(chan int, maxConcurrency)
const maxConcurrency = 16 const maxConcurrency = 16
var throttle = make(chan int, maxConcurrency)
func parseFlags() (locale string, deliverables []string, silent bool) { func parseFlags() (locale string, deliverables []string, silent bool) {
flag.StringVar( flag.StringVar(
&locale, "locale", "en-us", &locale, "locale", "en-us",
@ -172,6 +96,19 @@ func saveMainContent(toc *AtlasTOC) {
} }
} }
func saveContentVersion(toc *AtlasTOC) {
filePath := fmt.Sprintf("%s-version.txt", toc.Deliverable)
err := os.MkdirAll(filepath.Dir(filePath), 0755)
ExitIfError(err)
ofile, err := os.Create(filePath)
ExitIfError(err)
defer ofile.Close()
_, err = ofile.WriteString(toc.Version.DocVersion)
ExitIfError(err)
}
func main() { func main() {
locale, deliverables, silent := parseFlags() locale, deliverables, silent := parseFlags()
if silent { if silent {
@ -186,19 +123,19 @@ func main() {
} }
// Init the Sqlite db // Init the Sqlite db
dbmap = initDb() dbmap = InitDb()
err := dbmap.TruncateTables() err := dbmap.TruncateTables()
ExitIfError(err) ExitIfError(err)
for _, deliverable := range deliverables { for _, deliverable := range deliverables {
toc, err := getTOC(locale, deliverable) toc, err := getTOC(locale, deliverable)
ExitIfError(err)
saveMainContent(toc)
err = verifyVersion(toc) err = verifyVersion(toc)
WarnIfError(err) WarnIfError(err)
saveMainContent(toc)
saveContentVersion(toc)
// Download each entry // Download each entry
for _, entry := range toc.TOCEntries { for _, entry := range toc.TOCEntries {
if entry.ID == "apex_reference" || entry.ID == "pages_compref" { if entry.ID == "apex_reference" || entry.ID == "pages_compref" {
@ -319,78 +256,6 @@ var supportedTypes = []SupportedType{
}, },
} }
// IsType indicates that the TOCEntry is of a given SupportedType
// This is done by checking the suffix of the entry text
func (entry TOCEntry) IsType(t SupportedType) bool {
return strings.HasSuffix(entry.Text, t.TitleSuffix)
}
// CleanTitle trims known suffix from TOCEntry titles
func (entry TOCEntry) CleanTitle(t SupportedType) string {
if t.NoTrim {
return entry.Text
}
return strings.TrimSuffix(entry.Text, " "+t.TitleSuffix)
}
// GetRelLink extracts only the relative link from the Link Href
func (entry TOCEntry) GetRelLink(removeAnchor bool) (relLink string) {
if entry.LinkAttr.Href == "" {
return
}
// Get the JSON file
relLink = entry.LinkAttr.Href
if removeAnchor {
anchorIndex := strings.LastIndex(relLink, "#")
if anchorIndex > 0 {
relLink = relLink[0:anchorIndex]
}
}
return
}
// GetContent retrieves Content for this TOCEntry from the API
func (entry TOCEntry) GetContent(toc *AtlasTOC) (content *TOCContent, err error) {
relLink := entry.GetRelLink(true)
if relLink == "" {
return
}
url := fmt.Sprintf(
"https://developer.salesforce.com/docs/get_document_content/%s/%s/%s/%s",
toc.Deliverable,
relLink,
toc.Locale,
toc.Version.DocVersion,
)
// fmt.Println(url)
resp, err := http.Get(url)
if err != nil {
return
}
// Read the downloaded JSON
defer resp.Body.Close()
contents, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
// Load into Struct
content = new(TOCContent)
err = json.Unmarshal([]byte(contents), content)
if err != nil {
fmt.Println("Error reading JSON")
fmt.Println(resp.Status)
fmt.Println(url)
fmt.Println(string(contents))
return
}
return
}
func getEntryType(entry TOCEntry) (*SupportedType, error) { func getEntryType(entry TOCEntry) (*SupportedType, error) {
if strings.HasPrefix(entry.ID, "pages_compref_") { if strings.HasPrefix(entry.ID, "pages_compref_") {
return &SupportedType{ return &SupportedType{
@ -425,9 +290,9 @@ func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *Atlas
childType, err = getEntryType(child) childType, err = getEntryType(child)
if childType == nil && (entryType != nil && entryType.IsContainer) { if childType == nil && (entryType != nil && entryType.IsContainer) {
saveSearchIndex(dbmap, child, entryType, toc) SaveSearchIndex(dbmap, child, entryType, toc)
} else if childType != nil && !childType.IsContainer { } else if childType != nil && !childType.IsContainer {
saveSearchIndex(dbmap, child, childType, toc) SaveSearchIndex(dbmap, child, childType, toc)
} else { } else {
WarnIfError(err) WarnIfError(err)
} }
@ -443,16 +308,6 @@ func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *Atlas
} }
} }
// GetContentFilepath returns the filepath that should be used for the content
func (entry TOCEntry) GetContentFilepath(toc *AtlasTOC, removeAnchor bool) string {
relLink := entry.GetRelLink(removeAnchor)
if relLink == "" {
ExitIfError(NewFormatedError("Link not found for %s", entry.ID))
}
return fmt.Sprintf("atlas.%s.%s.meta/%s/%s", toc.Locale, toc.Deliverable, toc.Deliverable, relLink)
}
func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) { func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
@ -508,47 +363,3 @@ func downloadCSS(fileName string, wg *sync.WaitGroup) {
<-throttle <-throttle
} }
/**********************
Database
**********************/
func saveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
if entry.LinkAttr.Href == "" || entryType == nil {
return
}
relLink := entry.GetContentFilepath(toc, false)
name := entry.CleanTitle(*entryType)
if entryType.ShowNamespace && len(entryHierarchy) > 0 {
// Show namespace for methods
name = entryHierarchy[len(entryHierarchy)-1] + "." + name
}
// fmt.Println("Storing: " + name)
si := SearchIndex{
Name: name,
Type: entryType.TypeName,
Path: relLink,
}
dbmap.Insert(&si)
}
func initDb() *gorp.DbMap {
db, err := sql.Open("sqlite3", "docSet.dsidx")
ExitIfError(err)
dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}
dbmap.AddTableWithName(SearchIndex{}, "searchIndex").SetKeys(true, "ID")
err = dbmap.CreateTablesIfNotExists()
ExitIfError(err)
err = dbmap.TruncateTables()
ExitIfError(err)
return dbmap
}

155
SFDashC/structs.go Normal file
View File

@ -0,0 +1,155 @@
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
)
// JSON Structs
// AtlasTOC represents the meta documenation from Salesforce
type AtlasTOC struct {
AvailableVersions []VersionInfo `json:"available_versions"`
Content string
ContentDocumentID string `json:"content_document_id"`
Deliverable string
DocTitle string `json:"doc_title"`
Locale string
Language LanguageInfo
PDFUrl string `json:"pdf_url"`
TOCEntries []TOCEntry `json:"toc"`
Title string
Version VersionInfo
}
// LanguageInfo contains information for linking and displaying the language
type LanguageInfo struct {
Label string
Locale string
URL string
}
// VersionInfo representes a Salesforce documentation version
type VersionInfo struct {
DocVersion string `json:"doc_version"`
ReleaseVersion string `json:"release_version"`
VersionText string `json:"version_text"`
VersionURL string `json:"version_url"`
}
// TOCEntry represents a single Table of Contents item
type TOCEntry struct {
Text string
ID string
LinkAttr LinkAttr `json:"a_attr,omitempty"`
Children []TOCEntry
ComputedFirstTopic bool
ComputedResetPageLayout bool
}
// LinkAttr represents all attributes bound to a link
type LinkAttr struct {
Href string
}
// TOCContent contains content information for a piece of documenation
type TOCContent struct {
ID string
Title string
Content string
}
// Sqlite Struct
// SearchIndex is the database table that indexes the docs
type SearchIndex struct {
ID int64 `db:id`
Name string `db:name`
Type string `db:type`
Path string `db:path`
}
// IsType indicates that the TOCEntry is of a given SupportedType
// This is done by checking the suffix of the entry text
func (entry TOCEntry) IsType(t SupportedType) bool {
return strings.HasSuffix(entry.Text, t.TitleSuffix)
}
// CleanTitle trims known suffix from TOCEntry titles
func (entry TOCEntry) CleanTitle(t SupportedType) string {
if t.NoTrim {
return entry.Text
}
return strings.TrimSuffix(entry.Text, " "+t.TitleSuffix)
}
// GetRelLink extracts only the relative link from the Link Href
func (entry TOCEntry) GetRelLink(removeAnchor bool) (relLink string) {
if entry.LinkAttr.Href == "" {
return
}
// Get the JSON file
relLink = entry.LinkAttr.Href
if removeAnchor {
anchorIndex := strings.LastIndex(relLink, "#")
if anchorIndex > 0 {
relLink = relLink[0:anchorIndex]
}
}
return
}
// GetContent retrieves Content for this TOCEntry from the API
func (entry TOCEntry) GetContent(toc *AtlasTOC) (content *TOCContent, err error) {
relLink := entry.GetRelLink(true)
if relLink == "" {
return
}
url := fmt.Sprintf(
"https://developer.salesforce.com/docs/get_document_content/%s/%s/%s/%s",
toc.Deliverable,
relLink,
toc.Locale,
toc.Version.DocVersion,
)
// fmt.Println(url)
resp, err := http.Get(url)
if err != nil {
return
}
// Read the downloaded JSON
defer resp.Body.Close()
contents, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
// Load into Struct
content = new(TOCContent)
err = json.Unmarshal([]byte(contents), content)
if err != nil {
fmt.Println("Error reading JSON")
fmt.Println(resp.Status)
fmt.Println(url)
fmt.Println(string(contents))
return
}
return
}
// GetContentFilepath returns the filepath that should be used for the content
func (entry TOCEntry) GetContentFilepath(toc *AtlasTOC, removeAnchor bool) string {
relLink := entry.GetRelLink(removeAnchor)
if relLink == "" {
ExitIfError(NewFormatedError("Link not found for %s", entry.ID))
}
return fmt.Sprintf("atlas.%s.%s.meta/%s/%s", toc.Locale, toc.Deliverable, toc.Deliverable, relLink)
}