diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b3d368e --- /dev/null +++ b/Makefile @@ -0,0 +1,44 @@ +default: complete + +complete: run-combined package-apex package-vf package-combined + +run-apex: + (cd SFDashC && go run *.go --silent apexcode) + +run-vf: + (cd SFDashC && go run *.go --silent pages) + +run-combined: + (cd SFDashC && go run *.go --silent apexcode pages) + +package-apex: + $(eval type = Apex) + $(eval package = Salesforce $(type).docset) + mkdir -p "$(package)/Contents/Resources/Documents" + cp -r SFDashC/atlas.en-us.200.0.apexcode.meta "$(package)/Contents/Resources/Documents/" + cp SFDashC/*.html "$(package)/Contents/Resources/Documents/" + cp SFDashC/Info-$(type).plist "$(package)/Contents/Info.plist" + cp SFDashC/docSet.dsidx "$(package)/Contents/Resources/" + +package-vf: + $(eval type = Pages) + $(eval package = Salesforce $(type).docset) + mkdir -p "$(package)/Contents/Resources/Documents" + cp -r SFDashC/atlas.en-us.200.0.pages.meta "$(package)/Contents/Resources/Documents/" + cp SFDashC/*.html "$(package)/Contents/Resources/Documents/" + cp SFDashC/Info-$(type).plist "$(package)/Contents/Info.plist" + cp SFDashC/docSet.dsidx "$(package)/Contents/Resources/" + +package-combined: + $(eval type = Combined) + $(eval package = Salesforce $(type).docset) + mkdir -p "$(package)/Contents/Resources/Documents" + cp -r SFDashC/*.meta "$(package)/Contents/Resources/Documents/" + cp SFDashC/*.html "$(package)/Contents/Resources/Documents/" + cp SFDashC/Info-$(type).plist "$(package)/Contents/Info.plist" + cp SFDashC/docSet.dsidx "$(package)/Contents/Resources/" + +clean: + rm -fr SFDashC/*.meta + rm -f SFDashC/docSet.dsidx + rm -fr *.docset diff --git a/README.md b/README.md new file mode 100644 index 0000000..9e9c952 --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +SFDashC +======= + +SFDashC is a go application for downloading and constructing Dash docsets from the Salesforce online documentation + +Everything is wrapped with a Makefile and can be completely built by simply executing: + + make + +That's it! + +It will generate 3 docsets: Salesforce Apex, Salesforce Visualforce, and Salesforce Combined + +Dependencies +------------ + +Currently these are not auto resolved. You must install the following: + * github.com/coopernurse/gorp + * github.com/mattn/go-sqlite3 diff --git a/SFDashC/Info-Apex.plist b/SFDashC/Info-Apex.plist new file mode 100644 index 0000000..887be53 --- /dev/null +++ b/SFDashC/Info-Apex.plist @@ -0,0 +1,16 @@ + + + + + CFBundleIdentifier + Salesforce Apex + CFBundleName + Salesforce Apex + DocSetPlatformFamily + apex + isDashDocset + + dashIndexFilePath + apexcode.html + + diff --git a/SFDashC/Info-Combined.plist b/SFDashC/Info-Combined.plist new file mode 100644 index 0000000..fae7ba4 --- /dev/null +++ b/SFDashC/Info-Combined.plist @@ -0,0 +1,17 @@ + + + + + CFBundleIdentifier + Salesforce + + CFBundleName + Salesforce + DocSetPlatformFamily + sfdc + isDashDocset + + dashIndexFilePath + index.htm + + diff --git a/SFDashC/Info-Pages.plist b/SFDashC/Info-Pages.plist new file mode 100644 index 0000000..0ffa501 --- /dev/null +++ b/SFDashC/Info-Pages.plist @@ -0,0 +1,16 @@ + + + + + CFBundleIdentifier + Salesforce Visualforce + CFBundleName + Salesforce Visualforce + DocSetPlatformFamily + vf + isDashDocset + + dashIndexFilePath + pages.html + + diff --git a/SFDashC/errors.go b/SFDashC/errors.go new file mode 100644 index 0000000..4b3aef0 --- /dev/null +++ b/SFDashC/errors.go @@ -0,0 +1,53 @@ +package main + +import ( + "fmt" + "os" +) + +var shouldWarn = true + +// Custom errors +type errorString struct { + message string +} + +// Error retrievies the Error message from the error +func (err errorString) Error() string { + return err.message +} + +// NoWarn disables all warning output +func WithoutWarning() { + shouldWarn = false +} + +// NewCustomError creates a custom error using a string as the message +func NewCustomError(message string) error { + return &errorString{message} +} + +// NewFormatedError creates a new error using Sprintf +func NewFormatedError(format string, a ...interface{}) error { + return NewCustomError(fmt.Sprintf(format, a...)) +} + +// NewTypeNotFoundError returns an error for a TOCEntry with an unknown type +func NewTypeNotFoundError(entry TOCEntry) error { + return NewFormatedError("Type not found : %s %s", entry.Text, entry.ID) +} + +// ExitIfError is a helper function for terminating if an error is not nil +func ExitIfError(err error) { + if err != nil { + fmt.Println("ERROR :", err) + os.Exit(1) + } +} + +// WarnIfError is a helper function for terminating if an error is not nil +func WarnIfError(err error) { + if err != nil && shouldWarn { + fmt.Println("WARNING :", err) + } +} diff --git a/SFDashC/index.htm b/SFDashC/index.htm new file mode 100644 index 0000000..d3106c1 --- /dev/null +++ b/SFDashC/index.htm @@ -0,0 +1,5 @@ + + + This is a joint docset + + diff --git a/SFDashC/main.go b/SFDashC/main.go new file mode 100644 index 0000000..afcac11 --- /dev/null +++ b/SFDashC/main.go @@ -0,0 +1,513 @@ +package main + +import ( + "database/sql" + "encoding/json" + "flag" + "fmt" + "github.com/coopernurse/gorp" + _ "github.com/mattn/go-sqlite3" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + "sync" +) + +/* +TODO: + - Move structs to own file + - Move db stuff to own file + - Stylesheets +*/ + +// CSS Paths +var cssBasePath = "https://developer.salesforce.com/resource/stylesheets" +var cssFiles = []string{"docs.min.css"} + +// JSON Structs + +// AtlasTOC represents the meta documenation from Salesforce +type AtlasTOC struct { + AvailableVersions []VersionInfo `json:"available_versions"` + Content string + ContentDocumentID string `json:"content_document_id"` + Deliverable string + DocTitle string `json:"doc_title"` + Locale string + Language LanguageInfo + PDFUrl string `json:"pdf_url"` + TOCEntries []TOCEntry `json:"toc"` + Title string + Version VersionInfo +} + +// LanguageInfo contains information for linking and displaying the language +type LanguageInfo struct { + Label string + Locale string + URL string +} + +// VersionInfo representes a Salesforce documentation version +type VersionInfo struct { + DocVersion string `json:"doc_version"` + ReleaseVersion string `json:"release_version"` + VersionText string `json:"version_text"` + VersionURL string `json:"version_url"` +} + +// TOCEntry represents a single Table of Contents item +type TOCEntry struct { + Text string + ID string + LinkAttr LinkAttr `json:"a_attr,omitempty"` + Children []TOCEntry + ComputedFirstTopic bool + ComputedResetPageLayout bool +} + +// LinkAttr represents all attributes bound to a link +type LinkAttr struct { + Href string +} + +// TOCContent contains content information for a piece of documenation +type TOCContent struct { + ID string + Title string + Content string +} + +// Sqlite Struct + +// SearchIndex is the database table that indexes the docs +type SearchIndex struct { + ID int64 `db:id` + Name string `db:name` + Type string `db:type` + Path string `db:path` +} + +var dbmap *gorp.DbMap +var wg sync.WaitGroup + +const maxConcurrency = 16 + +var throttle = make(chan int, maxConcurrency) + +func parseFlags() (locale string, deliverables []string, silent bool) { + flag.StringVar( + &locale, "locale", "en-us", + "locale to use for documentation (default: en-us)", + ) + flag.BoolVar( + &silent, "silent", false, "this flag supresses warning messages", + ) + flag.Parse() + + // All other args are for deliverables + // apexcode or pages + deliverables = flag.Args() + return +} + +// getTOC Retrieves the TOC JSON and Unmarshals it +func getTOC(locale string, deliverable string) (toc *AtlasTOC, err error) { + var tocURL = fmt.Sprintf("https://developer.salesforce.com/docs/get_document/atlas.%s.%s.meta", locale, deliverable) + resp, err := http.Get(tocURL) + if err != nil { + return + } + + // Read the downloaded JSON + defer resp.Body.Close() + contents, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + // Load into Struct + toc = new(AtlasTOC) + err = json.Unmarshal([]byte(contents), toc) + return +} + +// verifyVersion ensures that the version retrieved is the latest +func verifyVersion(toc *AtlasTOC) error { + currentVersion := toc.Version.DocVersion + topVersion := toc.AvailableVersions[0].DocVersion + if currentVersion != topVersion { + return NewFormatedError("verifyVersion : retrieved version is not the latest. Found %s, latest is %s", currentVersion, topVersion) + } + return nil +} + +func printSuccess(toc *AtlasTOC) { + fmt.Println("Success:", toc.DocTitle, "-", toc.Version.VersionText) +} + +func saveMainContent(toc *AtlasTOC) { + filePath := fmt.Sprintf("%s.html", toc.Deliverable) + // Make sure file doesn't exist first + if _, err := os.Stat(filePath); os.IsNotExist(err) { + content := toc.Content + + err = os.MkdirAll(filepath.Dir(filePath), 0755) + ExitIfError(err) + + // TODO: Do something to format full page here + + ofile, err := os.Create(filePath) + ExitIfError(err) + + defer ofile.Close() + _, err = ofile.WriteString(content) + ExitIfError(err) + } +} + +func main() { + locale, deliverables, silent := parseFlags() + if silent { + WithoutWarning() + } + + // Init the Sqlite db + dbmap = initDb() + err := dbmap.TruncateTables() + ExitIfError(err) + + for _, deliverable := range deliverables { + toc, err := getTOC(locale, deliverable) + ExitIfError(err) + + saveMainContent(toc) + + err = verifyVersion(toc) + WarnIfError(err) + + // Download each entry + for _, entry := range toc.TOCEntries { + if entry.ID == "apex_reference" || entry.ID == "pages_compref" { + processChildReferences(entry, nil, toc) + } + } + + printSuccess(toc) + } + + // Download CSS + throttle <- 1 + /* + * wg.Add(1) + + * for _, cssUrl := range cssFiles { + * go downloadLink(cssBasePath+"/"+cssUrl, &wg) + * } + */ + + wg.Wait() +} + +// SupportedType contains information for generating indexes for types we care about +type SupportedType struct { + TypeName, TitleSuffix string + PushName, AppendParents, IsContainer, NoTrim, ShowNamespace, ParseContent bool +} + +var supportedTypes = []SupportedType{ + SupportedType{ + TypeName: "Method", + TitleSuffix: "Methods", + AppendParents: true, + IsContainer: true, + ShowNamespace: true, + }, + SupportedType{ + TypeName: "Constructor", + TitleSuffix: "Constructors", + AppendParents: true, + IsContainer: true, + ShowNamespace: false, + }, + SupportedType{ + TypeName: "Class", + TitleSuffix: "Class", + PushName: true, + AppendParents: true, + ShowNamespace: false, + }, + SupportedType{ + TypeName: "Namespace", + TitleSuffix: "Namespace", + PushName: true, + AppendParents: true, + ShowNamespace: false, + }, + SupportedType{ + TypeName: "Interface", + TitleSuffix: "Interface", + PushName: true, + AppendParents: true, + ShowNamespace: false, + }, + SupportedType{ + TypeName: "Statement", + TitleSuffix: "Statement", + ShowNamespace: false, + }, + SupportedType{ + TypeName: "Enum", + TitleSuffix: "Enum", + AppendParents: true, + ShowNamespace: false, + }, + SupportedType{ + TypeName: "Property", + TitleSuffix: "Properties", + AppendParents: true, + IsContainer: true, + ShowNamespace: false, + }, + SupportedType{ + TypeName: "Guide", + TitleSuffix: "Example Implementation", + NoTrim: true, + ShowNamespace: false, + }, + SupportedType{ + TypeName: "Statement", + TitleSuffix: "Statements", + NoTrim: true, + AppendParents: false, + IsContainer: true, + ShowNamespace: false, + }, + SupportedType{ + TypeName: "Field", + TitleSuffix: "Fields", + AppendParents: true, + PushName: true, + IsContainer: true, + ShowNamespace: false, + }, + SupportedType{ + TypeName: "Exception", + TitleSuffix: "Exceptions", + NoTrim: true, + AppendParents: true, + ShowNamespace: false, + ParseContent: true, + }, + SupportedType{ + TypeName: "Constant", + TitleSuffix: "Constants", + NoTrim: true, + AppendParents: true, + ShowNamespace: false, + ParseContent: true, + }, + SupportedType{ + TypeName: "Class", + TitleSuffix: "Class (Base Email Methods)", + PushName: true, + AppendParents: true, + ShowNamespace: false, + }, +} + +// IsType indicates that the TOCEntry is of a given SupportedType +// This is done by checking the suffix of the entry text +func (entry TOCEntry) IsType(t SupportedType) bool { + return strings.HasSuffix(entry.Text, t.TitleSuffix) +} + +// CleanTitle trims known suffix from TOCEntry titles +func (entry TOCEntry) CleanTitle(t SupportedType) string { + if t.NoTrim { + return entry.Text + } + return strings.TrimSuffix(entry.Text, " "+t.TitleSuffix) +} + +// GetRelLink extracts only the relative link from the Link Href +func (entry TOCEntry) GetRelLink() (relLink string) { + if entry.LinkAttr.Href == "" { + return + } + + // Get the JSON file + relLink = entry.LinkAttr.Href + anchorIndex := strings.LastIndex(relLink, "#") + if anchorIndex > 0 { + relLink = relLink[0:anchorIndex] + } + return +} + +// GetContent retrieves Content for this TOCEntry from the API +func (entry TOCEntry) GetContent(toc *AtlasTOC) (content *TOCContent, err error) { + relLink := entry.GetRelLink() + if relLink == "" { + return + } + + url := fmt.Sprintf( + "https://developer.salesforce.com/docs/get_document_content/%s/%s/%s/%s", + toc.Deliverable, + relLink, + toc.Locale, + toc.Version.DocVersion, + ) + + // fmt.Println(url) + resp, err := http.Get(url) + if err != nil { + return + } + + // Read the downloaded JSON + defer resp.Body.Close() + contents, err := ioutil.ReadAll(resp.Body) + // fmt.Println(string(contents)) + if err != nil { + return + } + + // Load into Struct + content = new(TOCContent) + err = json.Unmarshal([]byte(contents), content) + return +} + +func getEntryType(entry TOCEntry) (*SupportedType, error) { + if strings.HasPrefix(entry.ID, "pages_compref_") { + return &SupportedType{ + TypeName: "Tag", + NoTrim: true, + }, nil + } + for _, t := range supportedTypes { + if entry.IsType(t) { + return &t, nil + } + } + return nil, NewTypeNotFoundError(entry) +} + +var entryHierarchy []string + +func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) { + if entryType != nil && entryType.PushName { + entryHierarchy = append(entryHierarchy, entry.CleanTitle(*entryType)) + } + + for _, child := range entry.Children { + // fmt.Println("Processing: " + child.Text) + var err error + var childType *SupportedType + if child.LinkAttr.Href != "" { + throttle <- 1 + wg.Add(1) + + go downloadContent(child, toc, &wg) + + childType, err = getEntryType(child) + if childType == nil && (entryType != nil && entryType.IsContainer) { + saveSearchIndex(dbmap, child, entryType, toc) + } else if childType != nil && !childType.IsContainer { + saveSearchIndex(dbmap, child, childType, toc) + } else { + WarnIfError(err) + } + } + if len(child.Children) > 0 { + processChildReferences(child, childType, toc) + } + } + // fmt.Println("Done processing children for " + entry.Text) + + if entryType != nil && entryType.PushName { + entryHierarchy = entryHierarchy[:len(entryHierarchy)-1] + } +} + +// GetContentFilepath returns the filepath that should be used for the content +func (entry TOCEntry) GetContentFilepath(toc *AtlasTOC) string { + relLink := entry.GetRelLink() + if relLink == "" { + ExitIfError(NewFormatedError("Link not found for %s", entry.ID)) + } + + return fmt.Sprintf("%s/%s/%s", toc.Version.VersionURL, toc.Deliverable, relLink) +} + +func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) { + defer wg.Done() + + filePath := entry.GetContentFilepath(toc) + // Make sure file doesn't exist first + if _, err := os.Stat(filePath); os.IsNotExist(err) { + content, err := entry.GetContent(toc) + ExitIfError(err) + + err = os.MkdirAll(filepath.Dir(filePath), 0755) + ExitIfError(err) + + // TODO: Do something to format full page here + + ofile, err := os.Create(filePath) + ExitIfError(err) + + defer ofile.Close() + _, err = ofile.WriteString("\n" + content.Content) + ExitIfError(err) + } + <-throttle +} + +/********************** + Database +**********************/ + +func saveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) { + if entry.LinkAttr.Href == "" || entryType == nil { + return + } + + relLink := entry.GetContentFilepath(toc) + name := entry.CleanTitle(*entryType) + if entryType.ShowNamespace && len(entryHierarchy) > 0 { + // Show namespace for methods + name = entryHierarchy[len(entryHierarchy)-1] + "." + name + } + + // fmt.Println("Storing: " + name) + + si := SearchIndex{ + Name: name, + Type: entryType.TypeName, + Path: relLink, + } + + dbmap.Insert(&si) +} + +func initDb() *gorp.DbMap { + db, err := sql.Open("sqlite3", "docSet.dsidx") + ExitIfError(err) + + dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}} + + dbmap.AddTableWithName(SearchIndex{}, "searchIndex").SetKeys(true, "ID") + + err = dbmap.CreateTablesIfNotExists() + ExitIfError(err) + + err = dbmap.TruncateTables() + ExitIfError(err) + + return dbmap +}