First version

This commit is contained in:
ViViDboarder 2016-02-10 12:08:26 -08:00
parent e4be96da8e
commit 87db2342aa
8 changed files with 683 additions and 0 deletions

44
Makefile Normal file
View File

@ -0,0 +1,44 @@
default: complete
complete: run-combined package-apex package-vf package-combined
run-apex:
(cd SFDashC && go run *.go --silent apexcode)
run-vf:
(cd SFDashC && go run *.go --silent pages)
run-combined:
(cd SFDashC && go run *.go --silent apexcode pages)
package-apex:
$(eval type = Apex)
$(eval package = Salesforce $(type).docset)
mkdir -p "$(package)/Contents/Resources/Documents"
cp -r SFDashC/atlas.en-us.200.0.apexcode.meta "$(package)/Contents/Resources/Documents/"
cp SFDashC/*.html "$(package)/Contents/Resources/Documents/"
cp SFDashC/Info-$(type).plist "$(package)/Contents/Info.plist"
cp SFDashC/docSet.dsidx "$(package)/Contents/Resources/"
package-vf:
$(eval type = Pages)
$(eval package = Salesforce $(type).docset)
mkdir -p "$(package)/Contents/Resources/Documents"
cp -r SFDashC/atlas.en-us.200.0.pages.meta "$(package)/Contents/Resources/Documents/"
cp SFDashC/*.html "$(package)/Contents/Resources/Documents/"
cp SFDashC/Info-$(type).plist "$(package)/Contents/Info.plist"
cp SFDashC/docSet.dsidx "$(package)/Contents/Resources/"
package-combined:
$(eval type = Combined)
$(eval package = Salesforce $(type).docset)
mkdir -p "$(package)/Contents/Resources/Documents"
cp -r SFDashC/*.meta "$(package)/Contents/Resources/Documents/"
cp SFDashC/*.html "$(package)/Contents/Resources/Documents/"
cp SFDashC/Info-$(type).plist "$(package)/Contents/Info.plist"
cp SFDashC/docSet.dsidx "$(package)/Contents/Resources/"
clean:
rm -fr SFDashC/*.meta
rm -f SFDashC/docSet.dsidx
rm -fr *.docset

19
README.md Normal file
View File

@ -0,0 +1,19 @@
SFDashC
=======
SFDashC is a go application for downloading and constructing Dash docsets from the Salesforce online documentation
Everything is wrapped with a Makefile and can be completely built by simply executing:
make
That's it!
It will generate 3 docsets: Salesforce Apex, Salesforce Visualforce, and Salesforce Combined
Dependencies
------------
Currently these are not auto resolved. You must install the following:
* github.com/coopernurse/gorp
* github.com/mattn/go-sqlite3

16
SFDashC/Info-Apex.plist Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleIdentifier</key>
<string>Salesforce Apex</string>
<key>CFBundleName</key>
<string>Salesforce Apex</string>
<key>DocSetPlatformFamily</key>
<string>apex</string>
<key>isDashDocset</key>
<true/>
<key>dashIndexFilePath</key>
<string>apexcode.html</string>
</dict>
</plist>

View File

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleIdentifier</key>
<string>Salesforce</string>
<!-- Displayed in list view -->
<key>CFBundleName</key>
<string>Salesforce</string>
<key>DocSetPlatformFamily</key>
<string>sfdc</string>
<key>isDashDocset</key>
<true/>
<key>dashIndexFilePath</key>
<string>index.htm</string>
</dict>
</plist>

16
SFDashC/Info-Pages.plist Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleIdentifier</key>
<string>Salesforce Visualforce</string>
<key>CFBundleName</key>
<string>Salesforce Visualforce</string>
<key>DocSetPlatformFamily</key>
<string>vf</string>
<key>isDashDocset</key>
<true/>
<key>dashIndexFilePath</key>
<string>pages.html</string>
</dict>
</plist>

53
SFDashC/errors.go Normal file
View File

@ -0,0 +1,53 @@
package main
import (
"fmt"
"os"
)
var shouldWarn = true
// Custom errors
type errorString struct {
message string
}
// Error retrievies the Error message from the error
func (err errorString) Error() string {
return err.message
}
// NoWarn disables all warning output
func WithoutWarning() {
shouldWarn = false
}
// NewCustomError creates a custom error using a string as the message
func NewCustomError(message string) error {
return &errorString{message}
}
// NewFormatedError creates a new error using Sprintf
func NewFormatedError(format string, a ...interface{}) error {
return NewCustomError(fmt.Sprintf(format, a...))
}
// NewTypeNotFoundError returns an error for a TOCEntry with an unknown type
func NewTypeNotFoundError(entry TOCEntry) error {
return NewFormatedError("Type not found : %s %s", entry.Text, entry.ID)
}
// ExitIfError is a helper function for terminating if an error is not nil
func ExitIfError(err error) {
if err != nil {
fmt.Println("ERROR :", err)
os.Exit(1)
}
}
// WarnIfError is a helper function for terminating if an error is not nil
func WarnIfError(err error) {
if err != nil && shouldWarn {
fmt.Println("WARNING :", err)
}
}

5
SFDashC/index.htm Normal file
View File

@ -0,0 +1,5 @@
<html>
<body>
This is a joint docset
</body>
</html>

513
SFDashC/main.go Normal file
View File

@ -0,0 +1,513 @@
package main
import (
"database/sql"
"encoding/json"
"flag"
"fmt"
"github.com/coopernurse/gorp"
_ "github.com/mattn/go-sqlite3"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
)
/*
TODO:
- Move structs to own file
- Move db stuff to own file
- Stylesheets
*/
// CSS Paths
var cssBasePath = "https://developer.salesforce.com/resource/stylesheets"
var cssFiles = []string{"docs.min.css"}
// JSON Structs
// AtlasTOC represents the meta documenation from Salesforce
type AtlasTOC struct {
AvailableVersions []VersionInfo `json:"available_versions"`
Content string
ContentDocumentID string `json:"content_document_id"`
Deliverable string
DocTitle string `json:"doc_title"`
Locale string
Language LanguageInfo
PDFUrl string `json:"pdf_url"`
TOCEntries []TOCEntry `json:"toc"`
Title string
Version VersionInfo
}
// LanguageInfo contains information for linking and displaying the language
type LanguageInfo struct {
Label string
Locale string
URL string
}
// VersionInfo representes a Salesforce documentation version
type VersionInfo struct {
DocVersion string `json:"doc_version"`
ReleaseVersion string `json:"release_version"`
VersionText string `json:"version_text"`
VersionURL string `json:"version_url"`
}
// TOCEntry represents a single Table of Contents item
type TOCEntry struct {
Text string
ID string
LinkAttr LinkAttr `json:"a_attr,omitempty"`
Children []TOCEntry
ComputedFirstTopic bool
ComputedResetPageLayout bool
}
// LinkAttr represents all attributes bound to a link
type LinkAttr struct {
Href string
}
// TOCContent contains content information for a piece of documenation
type TOCContent struct {
ID string
Title string
Content string
}
// Sqlite Struct
// SearchIndex is the database table that indexes the docs
type SearchIndex struct {
ID int64 `db:id`
Name string `db:name`
Type string `db:type`
Path string `db:path`
}
var dbmap *gorp.DbMap
var wg sync.WaitGroup
const maxConcurrency = 16
var throttle = make(chan int, maxConcurrency)
func parseFlags() (locale string, deliverables []string, silent bool) {
flag.StringVar(
&locale, "locale", "en-us",
"locale to use for documentation (default: en-us)",
)
flag.BoolVar(
&silent, "silent", false, "this flag supresses warning messages",
)
flag.Parse()
// All other args are for deliverables
// apexcode or pages
deliverables = flag.Args()
return
}
// getTOC Retrieves the TOC JSON and Unmarshals it
func getTOC(locale string, deliverable string) (toc *AtlasTOC, err error) {
var tocURL = fmt.Sprintf("https://developer.salesforce.com/docs/get_document/atlas.%s.%s.meta", locale, deliverable)
resp, err := http.Get(tocURL)
if err != nil {
return
}
// Read the downloaded JSON
defer resp.Body.Close()
contents, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
// Load into Struct
toc = new(AtlasTOC)
err = json.Unmarshal([]byte(contents), toc)
return
}
// verifyVersion ensures that the version retrieved is the latest
func verifyVersion(toc *AtlasTOC) error {
currentVersion := toc.Version.DocVersion
topVersion := toc.AvailableVersions[0].DocVersion
if currentVersion != topVersion {
return NewFormatedError("verifyVersion : retrieved version is not the latest. Found %s, latest is %s", currentVersion, topVersion)
}
return nil
}
func printSuccess(toc *AtlasTOC) {
fmt.Println("Success:", toc.DocTitle, "-", toc.Version.VersionText)
}
func saveMainContent(toc *AtlasTOC) {
filePath := fmt.Sprintf("%s.html", toc.Deliverable)
// Make sure file doesn't exist first
if _, err := os.Stat(filePath); os.IsNotExist(err) {
content := toc.Content
err = os.MkdirAll(filepath.Dir(filePath), 0755)
ExitIfError(err)
// TODO: Do something to format full page here
ofile, err := os.Create(filePath)
ExitIfError(err)
defer ofile.Close()
_, err = ofile.WriteString(content)
ExitIfError(err)
}
}
func main() {
locale, deliverables, silent := parseFlags()
if silent {
WithoutWarning()
}
// Init the Sqlite db
dbmap = initDb()
err := dbmap.TruncateTables()
ExitIfError(err)
for _, deliverable := range deliverables {
toc, err := getTOC(locale, deliverable)
ExitIfError(err)
saveMainContent(toc)
err = verifyVersion(toc)
WarnIfError(err)
// Download each entry
for _, entry := range toc.TOCEntries {
if entry.ID == "apex_reference" || entry.ID == "pages_compref" {
processChildReferences(entry, nil, toc)
}
}
printSuccess(toc)
}
// Download CSS
throttle <- 1
/*
* wg.Add(1)
* for _, cssUrl := range cssFiles {
* go downloadLink(cssBasePath+"/"+cssUrl, &wg)
* }
*/
wg.Wait()
}
// SupportedType contains information for generating indexes for types we care about
type SupportedType struct {
TypeName, TitleSuffix string
PushName, AppendParents, IsContainer, NoTrim, ShowNamespace, ParseContent bool
}
var supportedTypes = []SupportedType{
SupportedType{
TypeName: "Method",
TitleSuffix: "Methods",
AppendParents: true,
IsContainer: true,
ShowNamespace: true,
},
SupportedType{
TypeName: "Constructor",
TitleSuffix: "Constructors",
AppendParents: true,
IsContainer: true,
ShowNamespace: false,
},
SupportedType{
TypeName: "Class",
TitleSuffix: "Class",
PushName: true,
AppendParents: true,
ShowNamespace: false,
},
SupportedType{
TypeName: "Namespace",
TitleSuffix: "Namespace",
PushName: true,
AppendParents: true,
ShowNamespace: false,
},
SupportedType{
TypeName: "Interface",
TitleSuffix: "Interface",
PushName: true,
AppendParents: true,
ShowNamespace: false,
},
SupportedType{
TypeName: "Statement",
TitleSuffix: "Statement",
ShowNamespace: false,
},
SupportedType{
TypeName: "Enum",
TitleSuffix: "Enum",
AppendParents: true,
ShowNamespace: false,
},
SupportedType{
TypeName: "Property",
TitleSuffix: "Properties",
AppendParents: true,
IsContainer: true,
ShowNamespace: false,
},
SupportedType{
TypeName: "Guide",
TitleSuffix: "Example Implementation",
NoTrim: true,
ShowNamespace: false,
},
SupportedType{
TypeName: "Statement",
TitleSuffix: "Statements",
NoTrim: true,
AppendParents: false,
IsContainer: true,
ShowNamespace: false,
},
SupportedType{
TypeName: "Field",
TitleSuffix: "Fields",
AppendParents: true,
PushName: true,
IsContainer: true,
ShowNamespace: false,
},
SupportedType{
TypeName: "Exception",
TitleSuffix: "Exceptions",
NoTrim: true,
AppendParents: true,
ShowNamespace: false,
ParseContent: true,
},
SupportedType{
TypeName: "Constant",
TitleSuffix: "Constants",
NoTrim: true,
AppendParents: true,
ShowNamespace: false,
ParseContent: true,
},
SupportedType{
TypeName: "Class",
TitleSuffix: "Class (Base Email Methods)",
PushName: true,
AppendParents: true,
ShowNamespace: false,
},
}
// IsType indicates that the TOCEntry is of a given SupportedType
// This is done by checking the suffix of the entry text
func (entry TOCEntry) IsType(t SupportedType) bool {
return strings.HasSuffix(entry.Text, t.TitleSuffix)
}
// CleanTitle trims known suffix from TOCEntry titles
func (entry TOCEntry) CleanTitle(t SupportedType) string {
if t.NoTrim {
return entry.Text
}
return strings.TrimSuffix(entry.Text, " "+t.TitleSuffix)
}
// GetRelLink extracts only the relative link from the Link Href
func (entry TOCEntry) GetRelLink() (relLink string) {
if entry.LinkAttr.Href == "" {
return
}
// Get the JSON file
relLink = entry.LinkAttr.Href
anchorIndex := strings.LastIndex(relLink, "#")
if anchorIndex > 0 {
relLink = relLink[0:anchorIndex]
}
return
}
// GetContent retrieves Content for this TOCEntry from the API
func (entry TOCEntry) GetContent(toc *AtlasTOC) (content *TOCContent, err error) {
relLink := entry.GetRelLink()
if relLink == "" {
return
}
url := fmt.Sprintf(
"https://developer.salesforce.com/docs/get_document_content/%s/%s/%s/%s",
toc.Deliverable,
relLink,
toc.Locale,
toc.Version.DocVersion,
)
// fmt.Println(url)
resp, err := http.Get(url)
if err != nil {
return
}
// Read the downloaded JSON
defer resp.Body.Close()
contents, err := ioutil.ReadAll(resp.Body)
// fmt.Println(string(contents))
if err != nil {
return
}
// Load into Struct
content = new(TOCContent)
err = json.Unmarshal([]byte(contents), content)
return
}
func getEntryType(entry TOCEntry) (*SupportedType, error) {
if strings.HasPrefix(entry.ID, "pages_compref_") {
return &SupportedType{
TypeName: "Tag",
NoTrim: true,
}, nil
}
for _, t := range supportedTypes {
if entry.IsType(t) {
return &t, nil
}
}
return nil, NewTypeNotFoundError(entry)
}
var entryHierarchy []string
func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
if entryType != nil && entryType.PushName {
entryHierarchy = append(entryHierarchy, entry.CleanTitle(*entryType))
}
for _, child := range entry.Children {
// fmt.Println("Processing: " + child.Text)
var err error
var childType *SupportedType
if child.LinkAttr.Href != "" {
throttle <- 1
wg.Add(1)
go downloadContent(child, toc, &wg)
childType, err = getEntryType(child)
if childType == nil && (entryType != nil && entryType.IsContainer) {
saveSearchIndex(dbmap, child, entryType, toc)
} else if childType != nil && !childType.IsContainer {
saveSearchIndex(dbmap, child, childType, toc)
} else {
WarnIfError(err)
}
}
if len(child.Children) > 0 {
processChildReferences(child, childType, toc)
}
}
// fmt.Println("Done processing children for " + entry.Text)
if entryType != nil && entryType.PushName {
entryHierarchy = entryHierarchy[:len(entryHierarchy)-1]
}
}
// GetContentFilepath returns the filepath that should be used for the content
func (entry TOCEntry) GetContentFilepath(toc *AtlasTOC) string {
relLink := entry.GetRelLink()
if relLink == "" {
ExitIfError(NewFormatedError("Link not found for %s", entry.ID))
}
return fmt.Sprintf("%s/%s/%s", toc.Version.VersionURL, toc.Deliverable, relLink)
}
func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) {
defer wg.Done()
filePath := entry.GetContentFilepath(toc)
// Make sure file doesn't exist first
if _, err := os.Stat(filePath); os.IsNotExist(err) {
content, err := entry.GetContent(toc)
ExitIfError(err)
err = os.MkdirAll(filepath.Dir(filePath), 0755)
ExitIfError(err)
// TODO: Do something to format full page here
ofile, err := os.Create(filePath)
ExitIfError(err)
defer ofile.Close()
_, err = ofile.WriteString("<base href=\"../../\"/>\n" + content.Content)
ExitIfError(err)
}
<-throttle
}
/**********************
Database
**********************/
func saveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
if entry.LinkAttr.Href == "" || entryType == nil {
return
}
relLink := entry.GetContentFilepath(toc)
name := entry.CleanTitle(*entryType)
if entryType.ShowNamespace && len(entryHierarchy) > 0 {
// Show namespace for methods
name = entryHierarchy[len(entryHierarchy)-1] + "." + name
}
// fmt.Println("Storing: " + name)
si := SearchIndex{
Name: name,
Type: entryType.TypeName,
Path: relLink,
}
dbmap.Insert(&si)
}
func initDb() *gorp.DbMap {
db, err := sql.Open("sqlite3", "docSet.dsidx")
ExitIfError(err)
dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}
dbmap.AddTableWithName(SearchIndex{}, "searchIndex").SetKeys(true, "ID")
err = dbmap.CreateTablesIfNotExists()
ExitIfError(err)
err = dbmap.TruncateTables()
ExitIfError(err)
return dbmap
}