First version

Makefile

@@ -0,0 +1,44 @@
+default: complete
+
+complete: run-combined package-apex package-vf package-combined
+
+run-apex:
+	(cd SFDashC && go run *.go --silent apexcode)
+
+run-vf:
+	(cd SFDashC && go run *.go --silent pages)
+
+run-combined:
+	(cd SFDashC && go run *.go --silent apexcode pages)
+
+package-apex:
+	$(eval type = Apex)
+	$(eval package = Salesforce $(type).docset)
+	mkdir -p "$(package)/Contents/Resources/Documents"
+	cp -r SFDashC/atlas.en-us.200.0.apexcode.meta "$(package)/Contents/Resources/Documents/"
+	cp SFDashC/*.html "$(package)/Contents/Resources/Documents/"
+	cp SFDashC/Info-$(type).plist "$(package)/Contents/Info.plist"
+	cp SFDashC/docSet.dsidx "$(package)/Contents/Resources/"
+
+package-vf:
+	$(eval type = Pages)
+	$(eval package = Salesforce $(type).docset)
+	mkdir -p "$(package)/Contents/Resources/Documents"
+	cp -r SFDashC/atlas.en-us.200.0.pages.meta "$(package)/Contents/Resources/Documents/"
+	cp SFDashC/*.html "$(package)/Contents/Resources/Documents/"
+	cp SFDashC/Info-$(type).plist "$(package)/Contents/Info.plist"
+	cp SFDashC/docSet.dsidx "$(package)/Contents/Resources/"
+
+package-combined:
+	$(eval type = Combined)
+	$(eval package = Salesforce $(type).docset)
+	mkdir -p "$(package)/Contents/Resources/Documents"
+	cp -r SFDashC/*.meta "$(package)/Contents/Resources/Documents/"
+	cp SFDashC/*.html "$(package)/Contents/Resources/Documents/"
+	cp SFDashC/Info-$(type).plist "$(package)/Contents/Info.plist"
+	cp SFDashC/docSet.dsidx "$(package)/Contents/Resources/"
+
+clean:
+	rm -fr SFDashC/*.meta
+	rm -f SFDashC/docSet.dsidx
+	rm -fr *.docset

README.md

@@ -0,0 +1,19 @@
+SFDashC
+=======
+
+SFDashC is a go application for downloading and constructing Dash docsets from the Salesforce online documentation
+
+Everything is wrapped with a Makefile and can be completely built by simply executing:
+
+    make
+
+That's it!
+
+It will generate 3 docsets: Salesforce Apex, Salesforce Visualforce, and Salesforce Combined
+
+Dependencies
+------------
+
+Currently these are not auto resolved. You must install the following:
+    * github.com/coopernurse/gorp
+    * github.com/mattn/go-sqlite3

SFDashC/Info-Apex.plist

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleIdentifier</key>
+	<string>Salesforce Apex</string>
+	<key>CFBundleName</key>
+	<string>Salesforce Apex</string>
+	<key>DocSetPlatformFamily</key>
+	<string>apex</string>
+	<key>isDashDocset</key>
+	<true/>
+    <key>dashIndexFilePath</key>
+    <string>apexcode.html</string>
+</dict>
+</plist>

SFDashC/Info-Combined.plist

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleIdentifier</key>
+	<string>Salesforce</string>
+    <!-- Displayed in list view -->
+	<key>CFBundleName</key>
+	<string>Salesforce</string>
+	<key>DocSetPlatformFamily</key>
+	<string>sfdc</string>
+	<key>isDashDocset</key>
+	<true/>
+    <key>dashIndexFilePath</key>
+    <string>index.htm</string>
+</dict>
+</plist>

SFDashC/Info-Pages.plist

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleIdentifier</key>
+	<string>Salesforce Visualforce</string>
+	<key>CFBundleName</key>
+	<string>Salesforce Visualforce</string>
+	<key>DocSetPlatformFamily</key>
+	<string>vf</string>
+	<key>isDashDocset</key>
+	<true/>
+    <key>dashIndexFilePath</key>
+    <string>pages.html</string>
+</dict>
+</plist>

SFDashC/errors.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+var shouldWarn = true
+
+// Custom errors
+type errorString struct {
+	message string
+}
+
+// Error retrievies the Error message from the error
+func (err errorString) Error() string {
+	return err.message
+}
+
+// NoWarn disables all warning output
+func WithoutWarning() {
+	shouldWarn = false
+}
+
+// NewCustomError creates a custom error using a string as the message
+func NewCustomError(message string) error {
+	return &errorString{message}
+}
+
+// NewFormatedError creates a new error using Sprintf
+func NewFormatedError(format string, a ...interface{}) error {
+	return NewCustomError(fmt.Sprintf(format, a...))
+}
+
+// NewTypeNotFoundError returns an error for a TOCEntry with an unknown type
+func NewTypeNotFoundError(entry TOCEntry) error {
+	return NewFormatedError("Type not found : %s %s", entry.Text, entry.ID)
+}
+
+// ExitIfError is a helper function for terminating if an error is not nil
+func ExitIfError(err error) {
+	if err != nil {
+		fmt.Println("ERROR :", err)
+		os.Exit(1)
+	}
+}
+
+// WarnIfError is a helper function for terminating if an error is not nil
+func WarnIfError(err error) {
+	if err != nil && shouldWarn {
+		fmt.Println("WARNING :", err)
+	}
+}

SFDashC/index.htm

@@ -0,0 +1,5 @@
+<html>
+    <body>
+        This is a joint docset
+    </body>
+</html>

SFDashC/main.go

@@ -0,0 +1,513 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"github.com/coopernurse/gorp"
+	_ "github.com/mattn/go-sqlite3"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+)
+
+/*
+TODO:
+ - Move structs to own file
+ - Move db stuff to own file
+ - Stylesheets
+*/
+
+// CSS Paths
+var cssBasePath = "https://developer.salesforce.com/resource/stylesheets"
+var cssFiles = []string{"docs.min.css"}
+
+// JSON Structs
+
+// AtlasTOC represents the meta documenation from Salesforce
+type AtlasTOC struct {
+	AvailableVersions []VersionInfo `json:"available_versions"`
+	Content           string
+	ContentDocumentID string `json:"content_document_id"`
+	Deliverable       string
+	DocTitle          string `json:"doc_title"`
+	Locale            string
+	Language          LanguageInfo
+	PDFUrl            string     `json:"pdf_url"`
+	TOCEntries        []TOCEntry `json:"toc"`
+	Title             string
+	Version           VersionInfo
+}
+
+// LanguageInfo contains information for linking and displaying the language
+type LanguageInfo struct {
+	Label  string
+	Locale string
+	URL    string
+}
+
+// VersionInfo representes a Salesforce documentation version
+type VersionInfo struct {
+	DocVersion     string `json:"doc_version"`
+	ReleaseVersion string `json:"release_version"`
+	VersionText    string `json:"version_text"`
+	VersionURL     string `json:"version_url"`
+}
+
+// TOCEntry represents a single Table of Contents item
+type TOCEntry struct {
+	Text                    string
+	ID                      string
+	LinkAttr                LinkAttr `json:"a_attr,omitempty"`
+	Children                []TOCEntry
+	ComputedFirstTopic      bool
+	ComputedResetPageLayout bool
+}
+
+// LinkAttr represents all attributes bound to a link
+type LinkAttr struct {
+	Href string
+}
+
+// TOCContent contains content information for a piece of documenation
+type TOCContent struct {
+	ID      string
+	Title   string
+	Content string
+}
+
+// Sqlite Struct
+
+// SearchIndex is the database table that indexes the docs
+type SearchIndex struct {
+	ID   int64  `db:id`
+	Name string `db:name`
+	Type string `db:type`
+	Path string `db:path`
+}
+
+var dbmap *gorp.DbMap
+var wg sync.WaitGroup
+
+const maxConcurrency = 16
+
+var throttle = make(chan int, maxConcurrency)
+
+func parseFlags() (locale string, deliverables []string, silent bool) {
+	flag.StringVar(
+		&locale, "locale", "en-us",
+		"locale to use for documentation (default: en-us)",
+	)
+	flag.BoolVar(
+		&silent, "silent", false, "this flag supresses warning messages",
+	)
+	flag.Parse()
+
+	// All other args are for deliverables
+	// apexcode or pages
+	deliverables = flag.Args()
+	return
+}
+
+// getTOC Retrieves the TOC JSON and Unmarshals it
+func getTOC(locale string, deliverable string) (toc *AtlasTOC, err error) {
+	var tocURL = fmt.Sprintf("https://developer.salesforce.com/docs/get_document/atlas.%s.%s.meta", locale, deliverable)
+	resp, err := http.Get(tocURL)
+	if err != nil {
+		return
+	}
+
+	// Read the downloaded JSON
+	defer resp.Body.Close()
+	contents, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+
+	// Load into Struct
+	toc = new(AtlasTOC)
+	err = json.Unmarshal([]byte(contents), toc)
+	return
+}
+
+// verifyVersion ensures that the version retrieved is the latest
+func verifyVersion(toc *AtlasTOC) error {
+	currentVersion := toc.Version.DocVersion
+	topVersion := toc.AvailableVersions[0].DocVersion
+	if currentVersion != topVersion {
+		return NewFormatedError("verifyVersion : retrieved version is not the latest. Found %s, latest is %s", currentVersion, topVersion)
+	}
+	return nil
+}
+
+func printSuccess(toc *AtlasTOC) {
+	fmt.Println("Success:", toc.DocTitle, "-", toc.Version.VersionText)
+}
+
+func saveMainContent(toc *AtlasTOC) {
+	filePath := fmt.Sprintf("%s.html", toc.Deliverable)
+	// Make sure file doesn't exist first
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		content := toc.Content
+
+		err = os.MkdirAll(filepath.Dir(filePath), 0755)
+		ExitIfError(err)
+
+		// TODO: Do something to format full page here
+
+		ofile, err := os.Create(filePath)
+		ExitIfError(err)
+
+		defer ofile.Close()
+		_, err = ofile.WriteString(content)
+		ExitIfError(err)
+	}
+}
+
+func main() {
+	locale, deliverables, silent := parseFlags()
+	if silent {
+		WithoutWarning()
+	}
+
+	// Init the Sqlite db
+	dbmap = initDb()
+	err := dbmap.TruncateTables()
+	ExitIfError(err)
+
+	for _, deliverable := range deliverables {
+		toc, err := getTOC(locale, deliverable)
+		ExitIfError(err)
+
+		saveMainContent(toc)
+
+		err = verifyVersion(toc)
+		WarnIfError(err)
+
+		// Download each entry
+		for _, entry := range toc.TOCEntries {
+			if entry.ID == "apex_reference" || entry.ID == "pages_compref" {
+				processChildReferences(entry, nil, toc)
+			}
+		}
+
+		printSuccess(toc)
+	}
+
+	// Download CSS
+	throttle <- 1
+	/*
+	 * wg.Add(1)
+
+	 * for _, cssUrl := range cssFiles {
+	 * 	go downloadLink(cssBasePath+"/"+cssUrl, &wg)
+	 * }
+	 */
+
+	wg.Wait()
+}
+
+// SupportedType contains information for generating indexes for types we care about
+type SupportedType struct {
+	TypeName, TitleSuffix                                                     string
+	PushName, AppendParents, IsContainer, NoTrim, ShowNamespace, ParseContent bool
+}
+
+var supportedTypes = []SupportedType{
+	SupportedType{
+		TypeName:      "Method",
+		TitleSuffix:   "Methods",
+		AppendParents: true,
+		IsContainer:   true,
+		ShowNamespace: true,
+	},
+	SupportedType{
+		TypeName:      "Constructor",
+		TitleSuffix:   "Constructors",
+		AppendParents: true,
+		IsContainer:   true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Class",
+		TitleSuffix:   "Class",
+		PushName:      true,
+		AppendParents: true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Namespace",
+		TitleSuffix:   "Namespace",
+		PushName:      true,
+		AppendParents: true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Interface",
+		TitleSuffix:   "Interface",
+		PushName:      true,
+		AppendParents: true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Statement",
+		TitleSuffix:   "Statement",
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Enum",
+		TitleSuffix:   "Enum",
+		AppendParents: true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Property",
+		TitleSuffix:   "Properties",
+		AppendParents: true,
+		IsContainer:   true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Guide",
+		TitleSuffix:   "Example Implementation",
+		NoTrim:        true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Statement",
+		TitleSuffix:   "Statements",
+		NoTrim:        true,
+		AppendParents: false,
+		IsContainer:   true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Field",
+		TitleSuffix:   "Fields",
+		AppendParents: true,
+		PushName:      true,
+		IsContainer:   true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Exception",
+		TitleSuffix:   "Exceptions",
+		NoTrim:        true,
+		AppendParents: true,
+		ShowNamespace: false,
+		ParseContent:  true,
+	},
+	SupportedType{
+		TypeName:      "Constant",
+		TitleSuffix:   "Constants",
+		NoTrim:        true,
+		AppendParents: true,
+		ShowNamespace: false,
+		ParseContent:  true,
+	},
+	SupportedType{
+		TypeName:      "Class",
+		TitleSuffix:   "Class (Base Email Methods)",
+		PushName:      true,
+		AppendParents: true,
+		ShowNamespace: false,
+	},
+}
+
+// IsType indicates that the TOCEntry is of a given SupportedType
+// This is done by checking the suffix of the entry text
+func (entry TOCEntry) IsType(t SupportedType) bool {
+	return strings.HasSuffix(entry.Text, t.TitleSuffix)
+}
+
+// CleanTitle trims known suffix from TOCEntry titles
+func (entry TOCEntry) CleanTitle(t SupportedType) string {
+	if t.NoTrim {
+		return entry.Text
+	}
+	return strings.TrimSuffix(entry.Text, " "+t.TitleSuffix)
+}
+
+// GetRelLink extracts only the relative link from the Link Href
+func (entry TOCEntry) GetRelLink() (relLink string) {
+	if entry.LinkAttr.Href == "" {
+		return
+	}
+
+	// Get the JSON file
+	relLink = entry.LinkAttr.Href
+	anchorIndex := strings.LastIndex(relLink, "#")
+	if anchorIndex > 0 {
+		relLink = relLink[0:anchorIndex]
+	}
+	return
+}
+
+// GetContent retrieves Content for this TOCEntry from the API
+func (entry TOCEntry) GetContent(toc *AtlasTOC) (content *TOCContent, err error) {
+	relLink := entry.GetRelLink()
+	if relLink == "" {
+		return
+	}
+
+	url := fmt.Sprintf(
+		"https://developer.salesforce.com/docs/get_document_content/%s/%s/%s/%s",
+		toc.Deliverable,
+		relLink,
+		toc.Locale,
+		toc.Version.DocVersion,
+	)
+
+	// fmt.Println(url)
+	resp, err := http.Get(url)
+	if err != nil {
+		return
+	}
+
+	// Read the downloaded JSON
+	defer resp.Body.Close()
+	contents, err := ioutil.ReadAll(resp.Body)
+	// fmt.Println(string(contents))
+	if err != nil {
+		return
+	}
+
+	// Load into Struct
+	content = new(TOCContent)
+	err = json.Unmarshal([]byte(contents), content)
+	return
+}
+
+func getEntryType(entry TOCEntry) (*SupportedType, error) {
+	if strings.HasPrefix(entry.ID, "pages_compref_") {
+		return &SupportedType{
+			TypeName: "Tag",
+			NoTrim:   true,
+		}, nil
+	}
+	for _, t := range supportedTypes {
+		if entry.IsType(t) {
+			return &t, nil
+		}
+	}
+	return nil, NewTypeNotFoundError(entry)
+}
+
+var entryHierarchy []string
+
+func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
+	if entryType != nil && entryType.PushName {
+		entryHierarchy = append(entryHierarchy, entry.CleanTitle(*entryType))
+	}
+
+	for _, child := range entry.Children {
+		// fmt.Println("Processing: " + child.Text)
+		var err error
+		var childType *SupportedType
+		if child.LinkAttr.Href != "" {
+			throttle <- 1
+			wg.Add(1)
+
+			go downloadContent(child, toc, &wg)
+
+			childType, err = getEntryType(child)
+			if childType == nil && (entryType != nil && entryType.IsContainer) {
+				saveSearchIndex(dbmap, child, entryType, toc)
+			} else if childType != nil && !childType.IsContainer {
+				saveSearchIndex(dbmap, child, childType, toc)
+			} else {
+				WarnIfError(err)
+			}
+		}
+		if len(child.Children) > 0 {
+			processChildReferences(child, childType, toc)
+		}
+	}
+	// fmt.Println("Done processing children for " + entry.Text)
+
+	if entryType != nil && entryType.PushName {
+		entryHierarchy = entryHierarchy[:len(entryHierarchy)-1]
+	}
+}
+
+// GetContentFilepath returns the filepath that should be used for the content
+func (entry TOCEntry) GetContentFilepath(toc *AtlasTOC) string {
+	relLink := entry.GetRelLink()
+	if relLink == "" {
+		ExitIfError(NewFormatedError("Link not found for %s", entry.ID))
+	}
+
+	return fmt.Sprintf("%s/%s/%s", toc.Version.VersionURL, toc.Deliverable, relLink)
+}
+
+func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	filePath := entry.GetContentFilepath(toc)
+	// Make sure file doesn't exist first
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		content, err := entry.GetContent(toc)
+		ExitIfError(err)
+
+		err = os.MkdirAll(filepath.Dir(filePath), 0755)
+		ExitIfError(err)
+
+		// TODO: Do something to format full page here
+
+		ofile, err := os.Create(filePath)
+		ExitIfError(err)
+
+		defer ofile.Close()
+		_, err = ofile.WriteString("<base href=\"../../\"/>\n" + content.Content)
+		ExitIfError(err)
+	}
+	<-throttle
+}
+
+/**********************
+    Database
+**********************/
+
+func saveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
+	if entry.LinkAttr.Href == "" || entryType == nil {
+		return
+	}
+
+	relLink := entry.GetContentFilepath(toc)
+	name := entry.CleanTitle(*entryType)
+	if entryType.ShowNamespace && len(entryHierarchy) > 0 {
+		// Show namespace for methods
+		name = entryHierarchy[len(entryHierarchy)-1] + "." + name
+	}
+
+	// fmt.Println("Storing: " + name)
+
+	si := SearchIndex{
+		Name: name,
+		Type: entryType.TypeName,
+		Path: relLink,
+	}
+
+	dbmap.Insert(&si)
+}
+
+func initDb() *gorp.DbMap {
+	db, err := sql.Open("sqlite3", "docSet.dsidx")
+	ExitIfError(err)
+
+	dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}
+
+	dbmap.AddTableWithName(SearchIndex{}, "searchIndex").SetKeys(true, "ID")
+
+	err = dbmap.CreateTablesIfNotExists()
+	ExitIfError(err)
+
+	err = dbmap.TruncateTables()
+	ExitIfError(err)
+
+	return dbmap
+}