diff --git a/.gitignore b/.gitignore index da1b3e7..7d641e7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ syntax-highlighter.min.css # External Go dependencies vendor/ build/ +.DS_Store diff --git a/Gopkg.lock b/Gopkg.lock index 8cf14d3..2547c26 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,4 +1,5 @@ -memo = "1f1d5928eb9c306d42831a9b9d96e1ef8d11095cd843a3222acd869ee999f650" +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + [[projects]] branch = "master" @@ -17,3 +18,10 @@ memo = "1f1d5928eb9c306d42831a9b9d96e1ef8d11095cd843a3222acd869ee999f650" name = "golang.org/x/net" packages = ["context"] revision = "feeb485667d1fdabe727840fe00adc22431bc86e" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "1f1d5928eb9c306d42831a9b9d96e1ef8d11095cd843a3222acd869ee999f650" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 19982ca..0c59d9b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,60 +1,7 @@ - -## Gopkg.toml example (these lines may be deleted) - -## "required" lists a set of packages (not projects) that must be included in -## Gopkg.lock. This list is merged with the set of packages imported by the current -## project. Use it when your project needs a package it doesn't explicitly import - -## including "main" packages. -# required = ["github.com/user/thing/cmd/thing"] - -## "ignored" lists a set of packages (not projects) that are ignored when -## dep statically analyzes source code. Ignored packages can be in this project, -## or in a dependency. -# ignored = ["github.com/user/project/badpkg"] - -## Dependencies define constraints on dependent projects. They are respected by -## dep whether coming from the Gopkg.toml of the current project or a dependency. -# [[dependencies]] -## Required: the root import path of the project being constrained. -# name = "github.com/user/project" -# -## Recommended: the version constraint to enforce for the project. -## Only one of "branch", "version" or "revision" can be specified. -# version = "1.0.0" -# branch = "master" -# revision = "abc123" -# -## Optional: an alternate location (URL or import path) for the project's source. -# source = "https://github.com/myfork/package.git" - -## Overrides have the same structure as [[dependencies]], but supercede all -## [[dependencies]] declarations from all projects. Only the current project's -## [[overrides]] are applied. -## -## Overrides are a sledgehammer. Use them only as a last resort. -# [[overrides]] -## Required: the root import path of the project being constrained. -# name = "github.com/user/project" -# -## Optional: specifying a version constraint override will cause all other -## constraints on this project to be ignored; only the overriden constraint -## need be satisfied. -## Again, only one of "branch", "version" or "revision" can be specified. -# version = "1.0.0" -# branch = "master" -# revision = "abc123" -# -## Optional: specifying an alternate source location as an override will -## enforce that the alternate location is used for that project, regardless of -## what source location any dependent projects specify. -# source = "https://github.com/myfork/package.git" - - - -[[dependencies]] +[[constraint]] branch = "master" name = "github.com/coopernurse/gorp" -[[dependencies]] +[[constraint]] branch = "master" name = "github.com/mattn/go-sqlite3" diff --git a/Makefile b/Makefile index 8ef7cb8..3d99076 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ default: all -all: clean-index package-apex clean-index package-vf clean-index package-combined +all: clean-index package-apex clean-index package-vf clean-index package-lightning clean-index package-combined run-apex: clean-index dep ensure @@ -12,9 +12,13 @@ run-vf: clean-index dep ensure go run ./SFDashC/*.go pages +run-lightning: clean-index + dep ensure + go run ./SFDashC/*.go -debug lightning + run-combined: clean-index dep ensure - go run ./SFDashC/*.go apexcode pages + go run ./SFDashC/*.go apexcode pages lightning package-apex: run-apex $(eval name = Apex) @@ -42,6 +46,19 @@ package-vf: run-vf cp ./build/docSet.dsidx "$(package)/Contents/Resources/" @echo "Docset generated!" +package-lightning: run-lightning + $(eval name = Lightning) + $(eval package = Salesforce $(name).docset) + $(eval version = $(shell cat ./build/lightning-version.txt)) + cat ./SFDashC/docset-lightning.json | sed s/VERSION/$(version)/ > ./build/docset-lightning.json + mkdir -p "$(package)/Contents/Resources/Documents" + cp -r ./build/atlas.en-us.lightning.meta "$(package)/Contents/Resources/Documents/" + cp ./build/*.html "$(package)/Contents/Resources/Documents/" + cp ./build/*.css "$(package)/Contents/Resources/Documents/" + cp ./SFDashC/Info-$(name).plist "$(package)/Contents/Info.plist" + cp ./build/docSet.dsidx "$(package)/Contents/Resources/" + @echo "Docset generated!" + package-combined: run-combined $(eval name = Combined) $(eval package = Salesforce $(name).docset) diff --git a/SFDashC/Info-Apex.plist b/SFDashC/Info-Apex.plist index d827904..5984b91 100644 --- a/SFDashC/Info-Apex.plist +++ b/SFDashC/Info-Apex.plist @@ -1,18 +1,18 @@ - - CFBundleIdentifier - Salesforce Apex - CFBundleName - Salesforce Apex - DocSetPlatformFamily - apex - isDashDocset - - dashIndexFilePath - apexcode.html - DashDocSetFallbackURL - https://developer.salesforce.com/docs/ - + + CFBundleIdentifier + Salesforce Apex + CFBundleName + Salesforce Apex + DocSetPlatformFamily + apex + isDashDocset + + dashIndexFilePath + apexcode.html + DashDocSetFallbackURL + https://developer.salesforce.com/docs/ + diff --git a/SFDashC/Info-Combined.plist b/SFDashC/Info-Combined.plist index 05648b7..0b18d17 100644 --- a/SFDashC/Info-Combined.plist +++ b/SFDashC/Info-Combined.plist @@ -1,19 +1,19 @@ - - CFBundleIdentifier - Salesforce - - CFBundleName - Salesforce - DocSetPlatformFamily - sfdc - isDashDocset - - dashIndexFilePath - index.htm - DashDocSetFallbackURL - https://developer.salesforce.com/docs/ - + + CFBundleIdentifier + Salesforce + + CFBundleName + Salesforce + DocSetPlatformFamily + sfdc + isDashDocset + + dashIndexFilePath + index.htm + DashDocSetFallbackURL + https://developer.salesforce.com/docs/ + diff --git a/SFDashC/Info-Lightning.plist b/SFDashC/Info-Lightning.plist new file mode 100644 index 0000000..b7ed4f4 --- /dev/null +++ b/SFDashC/Info-Lightning.plist @@ -0,0 +1,18 @@ + + + + + CFBundleIdentifier + Salesforce Lightning + CFBundleName + Salesforce Lightning + DocSetPlatformFamily + lightning + isDashDocset + + dashIndexFilePath + lightning.html + DashDocSetFallbackURL + https://developer.salesforce.com/docs/ + + diff --git a/SFDashC/Info-Pages.plist b/SFDashC/Info-Pages.plist index 8ffc812..b483e45 100644 --- a/SFDashC/Info-Pages.plist +++ b/SFDashC/Info-Pages.plist @@ -1,18 +1,18 @@ - - CFBundleIdentifier - Salesforce Visualforce - CFBundleName - Salesforce Visualforce - DocSetPlatformFamily - vf - isDashDocset - - dashIndexFilePath - pages.html - DashDocSetFallbackURL - https://developer.salesforce.com/docs/ - + + CFBundleIdentifier + Salesforce Visualforce + CFBundleName + Salesforce Visualforce + DocSetPlatformFamily + vf + isDashDocset + + dashIndexFilePath + pages.html + DashDocSetFallbackURL + https://developer.salesforce.com/docs/ + diff --git a/SFDashC/docset-apexcode.json b/SFDashC/docset-apexcode.json index a08f473..7beb7a3 100644 --- a/SFDashC/docset-apexcode.json +++ b/SFDashC/docset-apexcode.json @@ -8,7 +8,7 @@ }, "aliases": [ "apex", - "salesforce", + "salesforce", "sfdc" ], "specific_versions": [] diff --git a/SFDashC/docset-lightning.json b/SFDashC/docset-lightning.json new file mode 100644 index 0000000..5c203b8 --- /dev/null +++ b/SFDashC/docset-lightning.json @@ -0,0 +1,15 @@ +{ + "name": "Salesforce Lightning", + "version": "VERSION", + "archive": "Salesforce_Lightning.tgz", + "author": { + "name": "ViViDboarder", + "link": "https://github.com/ViViDboarder" + }, + "aliases": [ + "lightning", + "salesforce", + "sfdc" + ], + "specific_versions": [] +} diff --git a/SFDashC/docset-pages.json b/SFDashC/docset-pages.json index 195e209..f81e33a 100644 --- a/SFDashC/docset-pages.json +++ b/SFDashC/docset-pages.json @@ -8,7 +8,7 @@ }, "aliases": [ "visualforce", - "salesforce", + "salesforce", "sfdc" ], "specific_versions": [] diff --git a/SFDashC/errors.go b/SFDashC/errors.go index f465f69..1abb01b 100644 --- a/SFDashC/errors.go +++ b/SFDashC/errors.go @@ -31,6 +31,6 @@ func ExitIfError(err error) { // WarnIfError is a helper function for terminating if an error is not nil func WarnIfError(err error) { if err != nil { - LogDebug(err.Error()) + LogWarning(err.Error()) } } diff --git a/SFDashC/logging.go b/SFDashC/logging.go index 71e7fc9..497d00e 100644 --- a/SFDashC/logging.go +++ b/SFDashC/logging.go @@ -8,9 +8,10 @@ import ( const ( prefix = "SFDashC" // Log Levels - ERROR = iota - INFO = iota - DEBUG = iota + ERROR = iota + WARNING = iota + INFO = iota + DEBUG = iota ) var logLevel int @@ -23,6 +24,8 @@ func getLevelText() string { switch logLevel { case ERROR: return "ERROR" + case WARNING: + return "WARNING" case INFO: return "INFO" case DEBUG: @@ -58,6 +61,12 @@ func LogError(format string, a ...interface{}) { Log(ERROR, format, a...) } +// LogInfo will print a warning message +// If the level is greater than the maximum log level, it will not print +func LogWarning(format string, a ...interface{}) { + Log(WARNING, format, a...) +} + // LogInfo will print an info message // If the level is greater than the maximum log level, it will not print func LogInfo(format string, a ...interface{}) { diff --git a/SFDashC/main.go b/SFDashC/main.go index 917f83e..3108e24 100644 --- a/SFDashC/main.go +++ b/SFDashC/main.go @@ -33,7 +33,7 @@ func parseFlags() (locale string, deliverables []string, debug bool) { flag.Parse() // All other args are for deliverables - // apexcode or pages + // apexcode, pages, or lightening deliverables = flag.Args() return } @@ -41,6 +41,7 @@ func parseFlags() (locale string, deliverables []string, debug bool) { // getTOC Retrieves the TOC JSON and Unmarshals it func getTOC(locale string, deliverable string) (toc *AtlasTOC, err error) { var tocURL = fmt.Sprintf("https://developer.salesforce.com/docs/get_document/atlas.%s.%s.meta", locale, deliverable) + LogDebug("TOC URL: %s", tocURL) resp, err := http.Get(tocURL) ExitIfError(err) @@ -51,13 +52,18 @@ func getTOC(locale string, deliverable string) (toc *AtlasTOC, err error) { // Load into Struct toc = new(AtlasTOC) + LogDebug("TOC JSON: %s", string(contents)) err = json.Unmarshal([]byte(contents), toc) return } // verifyVersion ensures that the version retrieved is the latest func verifyVersion(toc *AtlasTOC) error { + // jsonVersion, _ := json.Marshal(toc.Version) + // LogDebug("toc.Version" + string(jsonVersion)) currentVersion := toc.Version.DocVersion + // jsonAvailVersions, _ := json.Marshal(toc.AvailableVersions) + // LogDebug("toc.AvailableVersions" + string(jsonAvailVersions)) topVersion := toc.AvailableVersions[0].DocVersion if currentVersion != topVersion { return NewFormatedError("verifyVersion: retrieved version is not the latest. Found %s, latest is %s", currentVersion, topVersion) @@ -109,42 +115,35 @@ func saveContentVersion(toc *AtlasTOC) { ExitIfError(err) } -func main() { - locale, deliverables, debug := parseFlags() - if debug { - SetLogLevel(DEBUG) +func downloadCSS(fileName string, wg *sync.WaitGroup) { + downloadFile(cssBaseURL+"/"+fileName, fileName, wg) +} + +func downloadFile(url string, fileName string, wg *sync.WaitGroup) { + if wg != nil { + defer wg.Done() } - // Download CSS - for _, cssFile := range cssFiles { - throttle <- 1 - wg.Add(1) - go downloadCSS(cssFile, &wg) + filePath := filepath.Join(buildDir, fileName) + if _, err := os.Stat(filePath); os.IsNotExist(err) { + err = os.MkdirAll(filepath.Dir(filePath), 0755) + ExitIfError(err) + + ofile, err := os.Create(filePath) + ExitIfError(err) + defer ofile.Close() + + response, err := http.Get(url) + ExitIfError(err) + defer response.Body.Close() + + _, err = io.Copy(ofile, response.Body) + ExitIfError(err) } - // Init the Sqlite db - dbmap = InitDb(buildDir) - err := dbmap.TruncateTables() - ExitIfError(err) - - for _, deliverable := range deliverables { - toc, err := getTOC(locale, deliverable) - - err = verifyVersion(toc) - WarnIfError(err) - - saveMainContent(toc) - saveContentVersion(toc) - - // Download each entry - for _, entry := range toc.TOCEntries { - processChildReferences(entry, nil, toc) - } - - printSuccess(toc) + if wg != nil { + <-throttle } - - wg.Wait() } func getEntryType(entry TOCEntry) (*SupportedType, error) { @@ -156,36 +155,56 @@ func getEntryType(entry TOCEntry) (*SupportedType, error) { return nil, NewTypeNotFoundError(entry) } +// processEntryReference downloads html and indexes a toc item +func processEntryReference(entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) { + LogDebug("Processing: %s", entry.Text) + throttle <- 1 + wg.Add(1) + + go downloadContent(entry, toc, &wg) + + if entryType == nil { + LogDebug("No entry type for %s. Cannot index", entry.Text) + } else if entryType.IsContainer || entryType.IsHidden { + LogDebug("%s is a container or is hidden. Do not index", entry.Text) + } else { + SaveSearchIndex(dbmap, entry, entryType, toc) + } +} + +// entryHierarchy allows breadcrumb naming var entryHierarchy []string +// processChildReferences iterates through all child toc items, cascading types, and indexes them func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) { if entryType != nil && entryType.PushName { entryHierarchy = append(entryHierarchy, entry.CleanTitle(*entryType)) } for _, child := range entry.Children { - LogDebug("Processing: %s", child.Text) + LogDebug("Reading child: %s", child.Text) var err error var childType *SupportedType + // Skip anything without an HTML page if child.LinkAttr.Href != "" { - throttle <- 1 - wg.Add(1) - - go downloadContent(child, toc, &wg) - childType, err = getEntryType(child) if childType == nil && entryType != nil && (entryType.IsContainer || entryType.CascadeType) { + // No child type, and parent is set to cascade LogDebug("Parent was container or cascade, using parent type of %s", entryType.TypeName) childType = entryType + childType.IsContainer = false + } else if childType != nil && entryType != nil { + // We didn't cascade in full, but some features are still hereditary + if entryType.IsHidden { + childType.IsHidden = true + } } - - if childType == nil { + if childType == nil && err != nil { WarnIfError(err) - } else if !childType.IsContainer { - SaveSearchIndex(dbmap, child, childType, toc) - } else { - LogDebug("%s is a container. Do not index", child.Text) } + processEntryReference(child, childType, toc) + } else { + LogDebug("%s has no link. Skipping", child.Text) } if len(child.Children) > 0 { processChildReferences(child, childType, toc) @@ -198,6 +217,7 @@ func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *Atlas } } +// downloadContent will download the html file for a given entry func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) { defer wg.Done() @@ -233,26 +253,48 @@ func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) { <-throttle } -func downloadCSS(fileName string, wg *sync.WaitGroup) { - defer wg.Done() - - filePath := filepath.Join(buildDir, fileName) - if _, err := os.Stat(filePath); os.IsNotExist(err) { - err = os.MkdirAll(filepath.Dir(filePath), 0755) - ExitIfError(err) - - ofile, err := os.Create(filePath) - ExitIfError(err) - defer ofile.Close() - - cssURL := cssBaseURL + "/" + fileName - response, err := http.Get(cssURL) - ExitIfError(err) - defer response.Body.Close() - - _, err = io.Copy(ofile, response.Body) - ExitIfError(err) +func main() { + LogInfo("Starting...") + locale, deliverables, debug := parseFlags() + if debug { + SetLogLevel(DEBUG) } - <-throttle + // Download CSS + for _, cssFile := range cssFiles { + throttle <- 1 + wg.Add(1) + go downloadCSS(cssFile, &wg) + } + + // Download icon + go downloadFile("https://developer.salesforce.com/resources2/favicon.ico", "icon.ico", nil) + + // Init the Sqlite db + dbmap = InitDb(buildDir) + err := dbmap.TruncateTables() + ExitIfError(err) + + for _, deliverable := range deliverables { + toc, err := getTOC(locale, deliverable) + + err = verifyVersion(toc) + WarnIfError(err) + + saveMainContent(toc) + saveContentVersion(toc) + + // Download each entry + for _, entry := range toc.TOCEntries { + entryType, err := getEntryType(entry) + if entryType != nil && err == nil { + processEntryReference(entry, entryType, toc) + } + processChildReferences(entry, entryType, toc) + } + + printSuccess(toc) + } + + wg.Wait() } diff --git a/SFDashC/structs.go b/SFDashC/structs.go index 54a4433..9e704e2 100644 --- a/SFDashC/structs.go +++ b/SFDashC/structs.go @@ -78,8 +78,10 @@ type SupportedType struct { TypeName string // Not sure... AppendParents bool - // Indicates that this just contains other nodes and we don't want to index this + // Indicates that this just contains other nodes and we don't want to index this node IsContainer bool + // Indicates that this and all nodes underneith should be hidden + IsHidden bool // Skip trimming of suffix from title NoTrim bool // Not sure... diff --git a/SFDashC/supportedtypes.go b/SFDashC/supportedtypes.go index f5f2d7b..0e4c8e1 100644 --- a/SFDashC/supportedtypes.go +++ b/SFDashC/supportedtypes.go @@ -1,6 +1,25 @@ package main var SupportedTypes = []SupportedType{ + // ID Based overrides should come first + SupportedType{ + ID: "ref_tag_set_attr_intf", + TypeName: "Guide", + }, + SupportedType{ + ID: "namespaces_intro", + TypeName: "Guide", + }, + SupportedType{ + ID: "namespaces_using_organization", + TypeName: "Guide", + }, + SupportedType{ + TypeName: "Guide", + ID: "apex_intro_get_started", + CascadeType: true, + }, + // Apex types SupportedType{ TypeName: "Method", TitleSuffix: "Methods", @@ -113,6 +132,7 @@ var SupportedTypes = []SupportedType{ AppendParents: false, ShowNamespace: false, }, + // VF Types SupportedType{ IDPrefix: "pages_compref_", TypeName: "Tag", @@ -170,9 +190,63 @@ var SupportedTypes = []SupportedType{ TypeName: "Guide", IDPrefix: "pages_variables_operators", }, + // Aurora components + SupportedType{ + TypeName: "Tag", + ID: "aura_compref", + IsContainer: true, + CascadeType: true, + }, + SupportedType{ + TypeName: "Tag", + ID: "ref_messaging", + IsContainer: true, + CascadeType: true, + }, + SupportedType{ + TypeName: "Interface", + ID: "ref_interfaces", + IsContainer: true, + CascadeType: true, + }, + SupportedType{ + TypeName: "Event", + ID: "ref_events", + IsContainer: true, + CascadeType: true, + }, + SupportedType{ + TypeName: "Event", + ID: "ref_events_aura", + IsContainer: true, + CascadeType: true, + }, + SupportedType{ + TypeName: "Guide", + ID: "debug_intro", + }, + SupportedType{ + TypeName: "Guide", + ID: "components_using", + }, + SupportedType{ + TypeName: "Guide", + ID: "components_overview", + }, SupportedType{ TypeName: "Guide", - ID: "apex_intro_get_started", + IDPrefix: "qs_intro", + IsContainer: true, + CascadeType: true, + }, + SupportedType{ + TypeName: "Guide", + ID: "events_intro", + }, + SupportedType{ + TypeName: "Guide", + ID: "apps_intro", + IsHidden: true, CascadeType: true, }, }