mirror of
https://github.com/ViViDboarder/docset-sfdc.git
synced 2024-11-14 17:36:34 +00:00
514 lines
12 KiB
Go
514 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"github.com/coopernurse/gorp"
|
|
_ "github.com/mattn/go-sqlite3"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
)
|
|
|
|
/*
|
|
TODO:
|
|
- Move structs to own file
|
|
- Move db stuff to own file
|
|
- Stylesheets
|
|
*/
|
|
|
|
// CSS Paths
|
|
var cssBasePath = "https://developer.salesforce.com/resource/stylesheets"
|
|
var cssFiles = []string{"docs.min.css"}
|
|
|
|
// JSON Structs
|
|
|
|
// AtlasTOC represents the meta documenation from Salesforce
|
|
type AtlasTOC struct {
|
|
AvailableVersions []VersionInfo `json:"available_versions"`
|
|
Content string
|
|
ContentDocumentID string `json:"content_document_id"`
|
|
Deliverable string
|
|
DocTitle string `json:"doc_title"`
|
|
Locale string
|
|
Language LanguageInfo
|
|
PDFUrl string `json:"pdf_url"`
|
|
TOCEntries []TOCEntry `json:"toc"`
|
|
Title string
|
|
Version VersionInfo
|
|
}
|
|
|
|
// LanguageInfo contains information for linking and displaying the language
|
|
type LanguageInfo struct {
|
|
Label string
|
|
Locale string
|
|
URL string
|
|
}
|
|
|
|
// VersionInfo representes a Salesforce documentation version
|
|
type VersionInfo struct {
|
|
DocVersion string `json:"doc_version"`
|
|
ReleaseVersion string `json:"release_version"`
|
|
VersionText string `json:"version_text"`
|
|
VersionURL string `json:"version_url"`
|
|
}
|
|
|
|
// TOCEntry represents a single Table of Contents item
|
|
type TOCEntry struct {
|
|
Text string
|
|
ID string
|
|
LinkAttr LinkAttr `json:"a_attr,omitempty"`
|
|
Children []TOCEntry
|
|
ComputedFirstTopic bool
|
|
ComputedResetPageLayout bool
|
|
}
|
|
|
|
// LinkAttr represents all attributes bound to a link
|
|
type LinkAttr struct {
|
|
Href string
|
|
}
|
|
|
|
// TOCContent contains content information for a piece of documenation
|
|
type TOCContent struct {
|
|
ID string
|
|
Title string
|
|
Content string
|
|
}
|
|
|
|
// Sqlite Struct
|
|
|
|
// SearchIndex is the database table that indexes the docs
|
|
type SearchIndex struct {
|
|
ID int64 `db:id`
|
|
Name string `db:name`
|
|
Type string `db:type`
|
|
Path string `db:path`
|
|
}
|
|
|
|
var dbmap *gorp.DbMap
|
|
var wg sync.WaitGroup
|
|
|
|
const maxConcurrency = 16
|
|
|
|
var throttle = make(chan int, maxConcurrency)
|
|
|
|
func parseFlags() (locale string, deliverables []string, silent bool) {
|
|
flag.StringVar(
|
|
&locale, "locale", "en-us",
|
|
"locale to use for documentation (default: en-us)",
|
|
)
|
|
flag.BoolVar(
|
|
&silent, "silent", false, "this flag supresses warning messages",
|
|
)
|
|
flag.Parse()
|
|
|
|
// All other args are for deliverables
|
|
// apexcode or pages
|
|
deliverables = flag.Args()
|
|
return
|
|
}
|
|
|
|
// getTOC Retrieves the TOC JSON and Unmarshals it
|
|
func getTOC(locale string, deliverable string) (toc *AtlasTOC, err error) {
|
|
var tocURL = fmt.Sprintf("https://developer.salesforce.com/docs/get_document/atlas.%s.%s.meta", locale, deliverable)
|
|
resp, err := http.Get(tocURL)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
// Read the downloaded JSON
|
|
defer resp.Body.Close()
|
|
contents, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
// Load into Struct
|
|
toc = new(AtlasTOC)
|
|
err = json.Unmarshal([]byte(contents), toc)
|
|
return
|
|
}
|
|
|
|
// verifyVersion ensures that the version retrieved is the latest
|
|
func verifyVersion(toc *AtlasTOC) error {
|
|
currentVersion := toc.Version.DocVersion
|
|
topVersion := toc.AvailableVersions[0].DocVersion
|
|
if currentVersion != topVersion {
|
|
return NewFormatedError("verifyVersion : retrieved version is not the latest. Found %s, latest is %s", currentVersion, topVersion)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func printSuccess(toc *AtlasTOC) {
|
|
fmt.Println("Success:", toc.DocTitle, "-", toc.Version.VersionText)
|
|
}
|
|
|
|
func saveMainContent(toc *AtlasTOC) {
|
|
filePath := fmt.Sprintf("%s.html", toc.Deliverable)
|
|
// Make sure file doesn't exist first
|
|
if _, err := os.Stat(filePath); os.IsNotExist(err) {
|
|
content := toc.Content
|
|
|
|
err = os.MkdirAll(filepath.Dir(filePath), 0755)
|
|
ExitIfError(err)
|
|
|
|
// TODO: Do something to format full page here
|
|
|
|
ofile, err := os.Create(filePath)
|
|
ExitIfError(err)
|
|
|
|
defer ofile.Close()
|
|
_, err = ofile.WriteString(content)
|
|
ExitIfError(err)
|
|
}
|
|
}
|
|
|
|
func main() {
|
|
locale, deliverables, silent := parseFlags()
|
|
if silent {
|
|
WithoutWarning()
|
|
}
|
|
|
|
// Init the Sqlite db
|
|
dbmap = initDb()
|
|
err := dbmap.TruncateTables()
|
|
ExitIfError(err)
|
|
|
|
for _, deliverable := range deliverables {
|
|
toc, err := getTOC(locale, deliverable)
|
|
ExitIfError(err)
|
|
|
|
saveMainContent(toc)
|
|
|
|
err = verifyVersion(toc)
|
|
WarnIfError(err)
|
|
|
|
// Download each entry
|
|
for _, entry := range toc.TOCEntries {
|
|
if entry.ID == "apex_reference" || entry.ID == "pages_compref" {
|
|
processChildReferences(entry, nil, toc)
|
|
}
|
|
}
|
|
|
|
printSuccess(toc)
|
|
}
|
|
|
|
// Download CSS
|
|
throttle <- 1
|
|
/*
|
|
* wg.Add(1)
|
|
|
|
* for _, cssUrl := range cssFiles {
|
|
* go downloadLink(cssBasePath+"/"+cssUrl, &wg)
|
|
* }
|
|
*/
|
|
|
|
wg.Wait()
|
|
}
|
|
|
|
// SupportedType contains information for generating indexes for types we care about
|
|
type SupportedType struct {
|
|
TypeName, TitleSuffix string
|
|
PushName, AppendParents, IsContainer, NoTrim, ShowNamespace, ParseContent bool
|
|
}
|
|
|
|
var supportedTypes = []SupportedType{
|
|
SupportedType{
|
|
TypeName: "Method",
|
|
TitleSuffix: "Methods",
|
|
AppendParents: true,
|
|
IsContainer: true,
|
|
ShowNamespace: true,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Constructor",
|
|
TitleSuffix: "Constructors",
|
|
AppendParents: true,
|
|
IsContainer: true,
|
|
ShowNamespace: false,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Class",
|
|
TitleSuffix: "Class",
|
|
PushName: true,
|
|
AppendParents: true,
|
|
ShowNamespace: false,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Namespace",
|
|
TitleSuffix: "Namespace",
|
|
PushName: true,
|
|
AppendParents: true,
|
|
ShowNamespace: false,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Interface",
|
|
TitleSuffix: "Interface",
|
|
PushName: true,
|
|
AppendParents: true,
|
|
ShowNamespace: false,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Statement",
|
|
TitleSuffix: "Statement",
|
|
ShowNamespace: false,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Enum",
|
|
TitleSuffix: "Enum",
|
|
AppendParents: true,
|
|
ShowNamespace: false,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Property",
|
|
TitleSuffix: "Properties",
|
|
AppendParents: true,
|
|
IsContainer: true,
|
|
ShowNamespace: false,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Guide",
|
|
TitleSuffix: "Example Implementation",
|
|
NoTrim: true,
|
|
ShowNamespace: false,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Statement",
|
|
TitleSuffix: "Statements",
|
|
NoTrim: true,
|
|
AppendParents: false,
|
|
IsContainer: true,
|
|
ShowNamespace: false,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Field",
|
|
TitleSuffix: "Fields",
|
|
AppendParents: true,
|
|
PushName: true,
|
|
IsContainer: true,
|
|
ShowNamespace: false,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Exception",
|
|
TitleSuffix: "Exceptions",
|
|
NoTrim: true,
|
|
AppendParents: true,
|
|
ShowNamespace: false,
|
|
ParseContent: true,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Constant",
|
|
TitleSuffix: "Constants",
|
|
NoTrim: true,
|
|
AppendParents: true,
|
|
ShowNamespace: false,
|
|
ParseContent: true,
|
|
},
|
|
SupportedType{
|
|
TypeName: "Class",
|
|
TitleSuffix: "Class (Base Email Methods)",
|
|
PushName: true,
|
|
AppendParents: true,
|
|
ShowNamespace: false,
|
|
},
|
|
}
|
|
|
|
// IsType indicates that the TOCEntry is of a given SupportedType
|
|
// This is done by checking the suffix of the entry text
|
|
func (entry TOCEntry) IsType(t SupportedType) bool {
|
|
return strings.HasSuffix(entry.Text, t.TitleSuffix)
|
|
}
|
|
|
|
// CleanTitle trims known suffix from TOCEntry titles
|
|
func (entry TOCEntry) CleanTitle(t SupportedType) string {
|
|
if t.NoTrim {
|
|
return entry.Text
|
|
}
|
|
return strings.TrimSuffix(entry.Text, " "+t.TitleSuffix)
|
|
}
|
|
|
|
// GetRelLink extracts only the relative link from the Link Href
|
|
func (entry TOCEntry) GetRelLink() (relLink string) {
|
|
if entry.LinkAttr.Href == "" {
|
|
return
|
|
}
|
|
|
|
// Get the JSON file
|
|
relLink = entry.LinkAttr.Href
|
|
anchorIndex := strings.LastIndex(relLink, "#")
|
|
if anchorIndex > 0 {
|
|
relLink = relLink[0:anchorIndex]
|
|
}
|
|
return
|
|
}
|
|
|
|
// GetContent retrieves Content for this TOCEntry from the API
|
|
func (entry TOCEntry) GetContent(toc *AtlasTOC) (content *TOCContent, err error) {
|
|
relLink := entry.GetRelLink()
|
|
if relLink == "" {
|
|
return
|
|
}
|
|
|
|
url := fmt.Sprintf(
|
|
"https://developer.salesforce.com/docs/get_document_content/%s/%s/%s/%s",
|
|
toc.Deliverable,
|
|
relLink,
|
|
toc.Locale,
|
|
toc.Version.DocVersion,
|
|
)
|
|
|
|
// fmt.Println(url)
|
|
resp, err := http.Get(url)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
// Read the downloaded JSON
|
|
defer resp.Body.Close()
|
|
contents, err := ioutil.ReadAll(resp.Body)
|
|
// fmt.Println(string(contents))
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
// Load into Struct
|
|
content = new(TOCContent)
|
|
err = json.Unmarshal([]byte(contents), content)
|
|
return
|
|
}
|
|
|
|
func getEntryType(entry TOCEntry) (*SupportedType, error) {
|
|
if strings.HasPrefix(entry.ID, "pages_compref_") {
|
|
return &SupportedType{
|
|
TypeName: "Tag",
|
|
NoTrim: true,
|
|
}, nil
|
|
}
|
|
for _, t := range supportedTypes {
|
|
if entry.IsType(t) {
|
|
return &t, nil
|
|
}
|
|
}
|
|
return nil, NewTypeNotFoundError(entry)
|
|
}
|
|
|
|
var entryHierarchy []string
|
|
|
|
func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
|
|
if entryType != nil && entryType.PushName {
|
|
entryHierarchy = append(entryHierarchy, entry.CleanTitle(*entryType))
|
|
}
|
|
|
|
for _, child := range entry.Children {
|
|
// fmt.Println("Processing: " + child.Text)
|
|
var err error
|
|
var childType *SupportedType
|
|
if child.LinkAttr.Href != "" {
|
|
throttle <- 1
|
|
wg.Add(1)
|
|
|
|
go downloadContent(child, toc, &wg)
|
|
|
|
childType, err = getEntryType(child)
|
|
if childType == nil && (entryType != nil && entryType.IsContainer) {
|
|
saveSearchIndex(dbmap, child, entryType, toc)
|
|
} else if childType != nil && !childType.IsContainer {
|
|
saveSearchIndex(dbmap, child, childType, toc)
|
|
} else {
|
|
WarnIfError(err)
|
|
}
|
|
}
|
|
if len(child.Children) > 0 {
|
|
processChildReferences(child, childType, toc)
|
|
}
|
|
}
|
|
// fmt.Println("Done processing children for " + entry.Text)
|
|
|
|
if entryType != nil && entryType.PushName {
|
|
entryHierarchy = entryHierarchy[:len(entryHierarchy)-1]
|
|
}
|
|
}
|
|
|
|
// GetContentFilepath returns the filepath that should be used for the content
|
|
func (entry TOCEntry) GetContentFilepath(toc *AtlasTOC) string {
|
|
relLink := entry.GetRelLink()
|
|
if relLink == "" {
|
|
ExitIfError(NewFormatedError("Link not found for %s", entry.ID))
|
|
}
|
|
|
|
return fmt.Sprintf("%s/%s/%s", toc.Version.VersionURL, toc.Deliverable, relLink)
|
|
}
|
|
|
|
func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) {
|
|
defer wg.Done()
|
|
|
|
filePath := entry.GetContentFilepath(toc)
|
|
// Make sure file doesn't exist first
|
|
if _, err := os.Stat(filePath); os.IsNotExist(err) {
|
|
content, err := entry.GetContent(toc)
|
|
ExitIfError(err)
|
|
|
|
err = os.MkdirAll(filepath.Dir(filePath), 0755)
|
|
ExitIfError(err)
|
|
|
|
// TODO: Do something to format full page here
|
|
|
|
ofile, err := os.Create(filePath)
|
|
ExitIfError(err)
|
|
|
|
defer ofile.Close()
|
|
_, err = ofile.WriteString("<base href=\"../../\"/>\n" + content.Content)
|
|
ExitIfError(err)
|
|
}
|
|
<-throttle
|
|
}
|
|
|
|
/**********************
|
|
Database
|
|
**********************/
|
|
|
|
func saveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
|
|
if entry.LinkAttr.Href == "" || entryType == nil {
|
|
return
|
|
}
|
|
|
|
relLink := entry.GetContentFilepath(toc)
|
|
name := entry.CleanTitle(*entryType)
|
|
if entryType.ShowNamespace && len(entryHierarchy) > 0 {
|
|
// Show namespace for methods
|
|
name = entryHierarchy[len(entryHierarchy)-1] + "." + name
|
|
}
|
|
|
|
// fmt.Println("Storing: " + name)
|
|
|
|
si := SearchIndex{
|
|
Name: name,
|
|
Type: entryType.TypeName,
|
|
Path: relLink,
|
|
}
|
|
|
|
dbmap.Insert(&si)
|
|
}
|
|
|
|
func initDb() *gorp.DbMap {
|
|
db, err := sql.Open("sqlite3", "docSet.dsidx")
|
|
ExitIfError(err)
|
|
|
|
dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}
|
|
|
|
dbmap.AddTableWithName(SearchIndex{}, "searchIndex").SetKeys(true, "ID")
|
|
|
|
err = dbmap.CreateTablesIfNotExists()
|
|
ExitIfError(err)
|
|
|
|
err = dbmap.TruncateTables()
|
|
ExitIfError(err)
|
|
|
|
return dbmap
|
|
}
|