Add support for lightning and clean up some of the functions

.gitignore

@@ -14,3 +14,4 @@ syntax-highlighter.min.css
 # External Go dependencies
 vendor/
 build/
+.DS_Store

Gopkg.lock

@@ -1,4 +1,5 @@
-memo = "1f1d5928eb9c306d42831a9b9d96e1ef8d11095cd843a3222acd869ee999f650"
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
 
 [[projects]]
   branch = "master"
@@ -17,3 +18,10 @@ memo = "1f1d5928eb9c306d42831a9b9d96e1ef8d11095cd843a3222acd869ee999f650"
   name = "golang.org/x/net"
   packages = ["context"]
   revision = "feeb485667d1fdabe727840fe00adc22431bc86e"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "1f1d5928eb9c306d42831a9b9d96e1ef8d11095cd843a3222acd869ee999f650"
+  solver-name = "gps-cdcl"
+  solver-version = 1

Gopkg.toml

@@ -1,60 +1,7 @@
-
+[[constraint]]
-## Gopkg.toml example (these lines may be deleted)
-
-## "required" lists a set of packages (not projects) that must be included in
-## Gopkg.lock. This list is merged with the set of packages imported by the current
-## project. Use it when your project needs a package it doesn't explicitly import -
-## including "main" packages.
-# required = ["github.com/user/thing/cmd/thing"]
-
-## "ignored" lists a set of packages (not projects) that are ignored when
-## dep statically analyzes source code. Ignored packages can be in this project,
-## or in a dependency.
-# ignored = ["github.com/user/project/badpkg"]
-
-## Dependencies define constraints on dependent projects. They are respected by
-## dep whether coming from the Gopkg.toml of the current project or a dependency.
-# [[dependencies]]
-## Required: the root import path of the project being constrained.
-# name = "github.com/user/project"
-#
-## Recommended: the version constraint to enforce for the project.
-## Only one of "branch", "version" or "revision" can be specified.
-# version = "1.0.0"
-# branch = "master"
-# revision = "abc123"
-#
-## Optional: an alternate location (URL or import path) for the project's source.
-# source = "https://github.com/myfork/package.git"
-
-## Overrides have the same structure as [[dependencies]], but supercede all
-## [[dependencies]] declarations from all projects. Only the current project's
-## [[overrides]] are applied.
-##
-## Overrides are a sledgehammer. Use them only as a last resort.
-# [[overrides]]
-## Required: the root import path of the project being constrained.
-# name = "github.com/user/project"
-#
-## Optional: specifying a version constraint override will cause all other
-## constraints on this project to be ignored; only the overriden constraint
-## need be satisfied.
-## Again, only one of "branch", "version" or "revision" can be specified.
-# version = "1.0.0"
-# branch = "master"
-# revision = "abc123"
-#
-## Optional: specifying an alternate source location as an override will
-## enforce that the alternate location is used for that project, regardless of
-## what source location any dependent projects specify.
-# source = "https://github.com/myfork/package.git"
-
-
-
-[[dependencies]]
   branch = "master"
   name = "github.com/coopernurse/gorp"
 
-[[dependencies]]
+[[constraint]]
   branch = "master"
   name = "github.com/mattn/go-sqlite3"

Makefile

@@ -2,7 +2,7 @@
 
 default: all
 
-all: clean-index package-apex clean-index package-vf clean-index package-combined
+all: clean-index package-apex clean-index package-vf clean-index package-lightning clean-index package-combined
 
 run-apex: clean-index
 	dep ensure
@@ -12,9 +12,13 @@ run-vf: clean-index
 	dep ensure
 	go run ./SFDashC/*.go pages
 
+run-lightning: clean-index
+	dep ensure
+	go run ./SFDashC/*.go -debug lightning
+
 run-combined: clean-index
 	dep ensure
-	go run ./SFDashC/*.go apexcode pages
+	go run ./SFDashC/*.go apexcode pages lightning
 
 package-apex: run-apex
 	$(eval name = Apex)
@@ -42,6 +46,19 @@ package-vf: run-vf
 	cp ./build/docSet.dsidx "$(package)/Contents/Resources/"
 	@echo "Docset generated!"
 
+package-lightning: run-lightning
+	$(eval name = Lightning)
+	$(eval package = Salesforce $(name).docset)
+	$(eval version = $(shell cat ./build/lightning-version.txt))
+	cat ./SFDashC/docset-lightning.json | sed s/VERSION/$(version)/ > ./build/docset-lightning.json
+	mkdir -p "$(package)/Contents/Resources/Documents"
+	cp -r ./build/atlas.en-us.lightning.meta "$(package)/Contents/Resources/Documents/"
+	cp ./build/*.html "$(package)/Contents/Resources/Documents/"
+	cp ./build/*.css "$(package)/Contents/Resources/Documents/"
+	cp ./SFDashC/Info-$(name).plist "$(package)/Contents/Info.plist"
+	cp ./build/docSet.dsidx "$(package)/Contents/Resources/"
+	@echo "Docset generated!"
+
 package-combined: run-combined
 	$(eval name = Combined)
 	$(eval package = Salesforce $(name).docset)

SFDashC/Info-Apex.plist

@@ -1,18 +1,18 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
-<dict>
+    <dict>
-	<key>CFBundleIdentifier</key>
+        <key>CFBundleIdentifier</key>
-	<string>Salesforce Apex</string>
+        <string>Salesforce Apex</string>
-	<key>CFBundleName</key>
+        <key>CFBundleName</key>
-	<string>Salesforce Apex</string>
+        <string>Salesforce Apex</string>
-	<key>DocSetPlatformFamily</key>
+        <key>DocSetPlatformFamily</key>
-	<string>apex</string>
+        <string>apex</string>
-	<key>isDashDocset</key>
+        <key>isDashDocset</key>
-	<true/>
+        <true/>
-    <key>dashIndexFilePath</key>
+        <key>dashIndexFilePath</key>
-    <string>apexcode.html</string>
+        <string>apexcode.html</string>
-    <key>DashDocSetFallbackURL</key>
+        <key>DashDocSetFallbackURL</key>
-    <string>https://developer.salesforce.com/docs/</string>
+        <string>https://developer.salesforce.com/docs/</string>
-</dict>
+    </dict>
 </plist>

SFDashC/Info-Combined.plist

@@ -1,19 +1,19 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
-<dict>
+    <dict>
-	<key>CFBundleIdentifier</key>
+        <key>CFBundleIdentifier</key>
-	<string>Salesforce</string>
+        <string>Salesforce</string>
-    <!-- Displayed in list view -->
+        <!-- Displayed in list view -->
-	<key>CFBundleName</key>
+        <key>CFBundleName</key>
-	<string>Salesforce</string>
+        <string>Salesforce</string>
-	<key>DocSetPlatformFamily</key>
+        <key>DocSetPlatformFamily</key>
-	<string>sfdc</string>
+        <string>sfdc</string>
-	<key>isDashDocset</key>
+        <key>isDashDocset</key>
-	<true/>
+        <true/>
-    <key>dashIndexFilePath</key>
+        <key>dashIndexFilePath</key>
-    <string>index.htm</string>
+        <string>index.htm</string>
-    <key>DashDocSetFallbackURL</key>
+        <key>DashDocSetFallbackURL</key>
-    <string>https://developer.salesforce.com/docs/</string>
+        <string>https://developer.salesforce.com/docs/</string>
-</dict>
+    </dict>
 </plist>

SFDashC/Info-Lightning.plist

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+    <dict>
+        <key>CFBundleIdentifier</key>
+        <string>Salesforce Lightning</string>
+        <key>CFBundleName</key>
+        <string>Salesforce Lightning</string>
+        <key>DocSetPlatformFamily</key>
+        <string>lightning</string>
+        <key>isDashDocset</key>
+        <true/>
+        <key>dashIndexFilePath</key>
+        <string>lightning.html</string>
+        <key>DashDocSetFallbackURL</key>
+        <string>https://developer.salesforce.com/docs/</string>
+    </dict>
+</plist>

SFDashC/Info-Pages.plist

@@ -1,18 +1,18 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
-<dict>
+    <dict>
-	<key>CFBundleIdentifier</key>
+        <key>CFBundleIdentifier</key>
-	<string>Salesforce Visualforce</string>
+        <string>Salesforce Visualforce</string>
-	<key>CFBundleName</key>
+        <key>CFBundleName</key>
-	<string>Salesforce Visualforce</string>
+        <string>Salesforce Visualforce</string>
-	<key>DocSetPlatformFamily</key>
+        <key>DocSetPlatformFamily</key>
-	<string>vf</string>
+        <string>vf</string>
-	<key>isDashDocset</key>
+        <key>isDashDocset</key>
-	<true/>
+        <true/>
-    <key>dashIndexFilePath</key>
+        <key>dashIndexFilePath</key>
-    <string>pages.html</string>
+        <string>pages.html</string>
-    <key>DashDocSetFallbackURL</key>
+        <key>DashDocSetFallbackURL</key>
-    <string>https://developer.salesforce.com/docs/</string>
+        <string>https://developer.salesforce.com/docs/</string>
-</dict>
+    </dict>
 </plist>

SFDashC/docset-lightning.json

@@ -0,0 +1,15 @@
+{
+    "name": "Salesforce Lightning",
+    "version": "VERSION",
+    "archive": "Salesforce_Lightning.tgz",
+    "author": {
+        "name": "ViViDboarder",
+        "link": "https://github.com/ViViDboarder"
+    },
+    "aliases": [
+        "lightning",
+        "salesforce",
+        "sfdc"
+    ],
+    "specific_versions": []
+}

SFDashC/errors.go

@@ -31,6 +31,6 @@ func ExitIfError(err error) {
 // WarnIfError is a helper function for terminating if an error is not nil
 func WarnIfError(err error) {
 	if err != nil {
-		LogDebug(err.Error())
+		LogWarning(err.Error())
 	}
 }

SFDashC/logging.go

@@ -8,9 +8,10 @@ import (
 const (
 	prefix = "SFDashC"
 	// Log Levels
-	ERROR = iota
+	ERROR   = iota
-	INFO  = iota
+	WARNING = iota
-	DEBUG = iota
+	INFO    = iota
+	DEBUG   = iota
 )
 
 var logLevel int
@@ -23,6 +24,8 @@ func getLevelText() string {
 	switch logLevel {
 	case ERROR:
 		return "ERROR"
+	case WARNING:
+		return "WARNING"
 	case INFO:
 		return "INFO"
 	case DEBUG:
@@ -58,6 +61,12 @@ func LogError(format string, a ...interface{}) {
 	Log(ERROR, format, a...)
 }
 
+// LogInfo will print a warning message
+// If the level is greater than the maximum log level, it will not print
+func LogWarning(format string, a ...interface{}) {
+	Log(WARNING, format, a...)
+}
+
 // LogInfo will print an info message
 // If the level is greater than the maximum log level, it will not print
 func LogInfo(format string, a ...interface{}) {

SFDashC/main.go

@@ -33,7 +33,7 @@ func parseFlags() (locale string, deliverables []string, debug bool) {
 	flag.Parse()
 
 	// All other args are for deliverables
-	// apexcode or pages
+	// apexcode, pages, or lightening
 	deliverables = flag.Args()
 	return
 }
@@ -41,6 +41,7 @@ func parseFlags() (locale string, deliverables []string, debug bool) {
 // getTOC Retrieves the TOC JSON and Unmarshals it
 func getTOC(locale string, deliverable string) (toc *AtlasTOC, err error) {
 	var tocURL = fmt.Sprintf("https://developer.salesforce.com/docs/get_document/atlas.%s.%s.meta", locale, deliverable)
+	LogDebug("TOC URL: %s", tocURL)
 	resp, err := http.Get(tocURL)
 	ExitIfError(err)
 
@@ -51,13 +52,18 @@ func getTOC(locale string, deliverable string) (toc *AtlasTOC, err error) {
 
 	// Load into Struct
 	toc = new(AtlasTOC)
+	LogDebug("TOC JSON: %s", string(contents))
 	err = json.Unmarshal([]byte(contents), toc)
 	return
 }
 
 // verifyVersion ensures that the version retrieved is the latest
 func verifyVersion(toc *AtlasTOC) error {
+	// jsonVersion, _ := json.Marshal(toc.Version)
+	// LogDebug("toc.Version" + string(jsonVersion))
 	currentVersion := toc.Version.DocVersion
+	// jsonAvailVersions, _ := json.Marshal(toc.AvailableVersions)
+	// LogDebug("toc.AvailableVersions" + string(jsonAvailVersions))
 	topVersion := toc.AvailableVersions[0].DocVersion
 	if currentVersion != topVersion {
 		return NewFormatedError("verifyVersion: retrieved version is not the latest. Found %s, latest is %s", currentVersion, topVersion)
@@ -109,42 +115,35 @@ func saveContentVersion(toc *AtlasTOC) {
 	ExitIfError(err)
 }
 
-func main() {
+func downloadCSS(fileName string, wg *sync.WaitGroup) {
-	locale, deliverables, debug := parseFlags()
+	downloadFile(cssBaseURL+"/"+fileName, fileName, wg)
-	if debug {
+}
-		SetLogLevel(DEBUG)
+
+func downloadFile(url string, fileName string, wg *sync.WaitGroup) {
+	if wg != nil {
+		defer wg.Done()
 	}
 
-	// Download CSS
+	filePath := filepath.Join(buildDir, fileName)
-	for _, cssFile := range cssFiles {
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
-		throttle <- 1
+		err = os.MkdirAll(filepath.Dir(filePath), 0755)
-		wg.Add(1)
+		ExitIfError(err)
-		go downloadCSS(cssFile, &wg)
+
+		ofile, err := os.Create(filePath)
+		ExitIfError(err)
+		defer ofile.Close()
+
+		response, err := http.Get(url)
+		ExitIfError(err)
+		defer response.Body.Close()
+
+		_, err = io.Copy(ofile, response.Body)
+		ExitIfError(err)
 	}
 
-	// Init the Sqlite db
+	if wg != nil {
-	dbmap = InitDb(buildDir)
+		<-throttle
-	err := dbmap.TruncateTables()
-	ExitIfError(err)
-
-	for _, deliverable := range deliverables {
-		toc, err := getTOC(locale, deliverable)
-
-		err = verifyVersion(toc)
-		WarnIfError(err)
-
-		saveMainContent(toc)
-		saveContentVersion(toc)
-
-		// Download each entry
-		for _, entry := range toc.TOCEntries {
-			processChildReferences(entry, nil, toc)
-		}
-
-		printSuccess(toc)
 	}
-
-	wg.Wait()
 }
 
 func getEntryType(entry TOCEntry) (*SupportedType, error) {
@@ -156,36 +155,56 @@ func getEntryType(entry TOCEntry) (*SupportedType, error) {
 	return nil, NewTypeNotFoundError(entry)
 }
 
+// processEntryReference downloads html and indexes a toc item
+func processEntryReference(entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
+	LogDebug("Processing: %s", entry.Text)
+	throttle <- 1
+	wg.Add(1)
+
+	go downloadContent(entry, toc, &wg)
+
+	if entryType == nil {
+		LogDebug("No entry type for %s. Cannot index", entry.Text)
+	} else if entryType.IsContainer || entryType.IsHidden {
+		LogDebug("%s is a container or is hidden. Do not index", entry.Text)
+	} else {
+		SaveSearchIndex(dbmap, entry, entryType, toc)
+	}
+}
+
+// entryHierarchy allows breadcrumb naming
 var entryHierarchy []string
 
+// processChildReferences iterates through all child toc items, cascading types, and indexes them
 func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
 	if entryType != nil && entryType.PushName {
 		entryHierarchy = append(entryHierarchy, entry.CleanTitle(*entryType))
 	}
 
 	for _, child := range entry.Children {
-		LogDebug("Processing: %s", child.Text)
+		LogDebug("Reading child: %s", child.Text)
 		var err error
 		var childType *SupportedType
+		// Skip anything without an HTML page
 		if child.LinkAttr.Href != "" {
-			throttle <- 1
-			wg.Add(1)
-
-			go downloadContent(child, toc, &wg)
-
 			childType, err = getEntryType(child)
 			if childType == nil && entryType != nil && (entryType.IsContainer || entryType.CascadeType) {
+				// No child type, and parent is set to cascade
 				LogDebug("Parent was container or cascade, using parent type of %s", entryType.TypeName)
 				childType = entryType
+				childType.IsContainer = false
+			} else if childType != nil && entryType != nil {
+				// We didn't cascade in full, but some features are still hereditary
+				if entryType.IsHidden {
+					childType.IsHidden = true
+				}
 			}
-
+			if childType == nil && err != nil {
-			if childType == nil {
 				WarnIfError(err)
-			} else if !childType.IsContainer {
-				SaveSearchIndex(dbmap, child, childType, toc)
-			} else {
-				LogDebug("%s is a container. Do not index", child.Text)
 			}
+			processEntryReference(child, childType, toc)
+		} else {
+			LogDebug("%s has no link. Skipping", child.Text)
 		}
 		if len(child.Children) > 0 {
 			processChildReferences(child, childType, toc)
@@ -198,6 +217,7 @@ func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *Atlas
 	}
 }
 
+// downloadContent will download the html file for a given entry
 func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) {
 	defer wg.Done()
 
@@ -233,26 +253,48 @@ func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) {
 	<-throttle
 }
 
-func downloadCSS(fileName string, wg *sync.WaitGroup) {
+func main() {
-	defer wg.Done()
+	LogInfo("Starting...")
-
+	locale, deliverables, debug := parseFlags()
-	filePath := filepath.Join(buildDir, fileName)
+	if debug {
-	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		SetLogLevel(DEBUG)
-		err = os.MkdirAll(filepath.Dir(filePath), 0755)
-		ExitIfError(err)
-
-		ofile, err := os.Create(filePath)
-		ExitIfError(err)
-		defer ofile.Close()
-
-		cssURL := cssBaseURL + "/" + fileName
-		response, err := http.Get(cssURL)
-		ExitIfError(err)
-		defer response.Body.Close()
-
-		_, err = io.Copy(ofile, response.Body)
-		ExitIfError(err)
 	}
 
-	<-throttle
+	// Download CSS
+	for _, cssFile := range cssFiles {
+		throttle <- 1
+		wg.Add(1)
+		go downloadCSS(cssFile, &wg)
+	}
+
+	// Download icon
+	go downloadFile("https://developer.salesforce.com/resources2/favicon.ico", "icon.ico", nil)
+
+	// Init the Sqlite db
+	dbmap = InitDb(buildDir)
+	err := dbmap.TruncateTables()
+	ExitIfError(err)
+
+	for _, deliverable := range deliverables {
+		toc, err := getTOC(locale, deliverable)
+
+		err = verifyVersion(toc)
+		WarnIfError(err)
+
+		saveMainContent(toc)
+		saveContentVersion(toc)
+
+		// Download each entry
+		for _, entry := range toc.TOCEntries {
+			entryType, err := getEntryType(entry)
+			if entryType != nil && err == nil {
+				processEntryReference(entry, entryType, toc)
+			}
+			processChildReferences(entry, entryType, toc)
+		}
+
+		printSuccess(toc)
+	}
+
+	wg.Wait()
 }

SFDashC/structs.go

@@ -78,8 +78,10 @@ type SupportedType struct {
 	TypeName string
 	// Not sure...
 	AppendParents bool
-	// Indicates that this just contains other nodes and we don't want to index this
+	// Indicates that this just contains other nodes and we don't want to index this node
 	IsContainer bool
+	// Indicates that this and all nodes underneith should be hidden
+	IsHidden bool
 	// Skip trimming of suffix from title
 	NoTrim bool
 	// Not sure...

SFDashC/supportedtypes.go

@@ -1,6 +1,25 @@
 package main
 
 var SupportedTypes = []SupportedType{
+	// ID Based overrides should come first
+	SupportedType{
+		ID:       "ref_tag_set_attr_intf",
+		TypeName: "Guide",
+	},
+	SupportedType{
+		ID:       "namespaces_intro",
+		TypeName: "Guide",
+	},
+	SupportedType{
+		ID:       "namespaces_using_organization",
+		TypeName: "Guide",
+	},
+	SupportedType{
+		TypeName:    "Guide",
+		ID:          "apex_intro_get_started",
+		CascadeType: true,
+	},
+	// Apex types
 	SupportedType{
 		TypeName:      "Method",
 		TitleSuffix:   "Methods",
@@ -113,6 +132,7 @@ var SupportedTypes = []SupportedType{
 		AppendParents: false,
 		ShowNamespace: false,
 	},
+	// VF Types
 	SupportedType{
 		IDPrefix: "pages_compref_",
 		TypeName: "Tag",
@@ -170,9 +190,63 @@ var SupportedTypes = []SupportedType{
 		TypeName: "Guide",
 		IDPrefix: "pages_variables_operators",
 	},
+	// Aurora components
+	SupportedType{
+		TypeName:    "Tag",
+		ID:          "aura_compref",
+		IsContainer: true,
+		CascadeType: true,
+	},
+	SupportedType{
+		TypeName:    "Tag",
+		ID:          "ref_messaging",
+		IsContainer: true,
+		CascadeType: true,
+	},
+	SupportedType{
+		TypeName:    "Interface",
+		ID:          "ref_interfaces",
+		IsContainer: true,
+		CascadeType: true,
+	},
+	SupportedType{
+		TypeName:    "Event",
+		ID:          "ref_events",
+		IsContainer: true,
+		CascadeType: true,
+	},
+	SupportedType{
+		TypeName:    "Event",
+		ID:          "ref_events_aura",
+		IsContainer: true,
+		CascadeType: true,
+	},
+	SupportedType{
+		TypeName: "Guide",
+		ID:       "debug_intro",
+	},
+	SupportedType{
+		TypeName: "Guide",
+		ID:       "components_using",
+	},
+	SupportedType{
+		TypeName: "Guide",
+		ID:       "components_overview",
+	},
 	SupportedType{
 		TypeName:    "Guide",
-		ID:          "apex_intro_get_started",
+		IDPrefix:    "qs_intro",
+		IsContainer: true,
+		CascadeType: true,
+	},
+	SupportedType{
+		TypeName: "Guide",
+		ID:       "events_intro",
+	},
+	SupportedType{
+		TypeName:    "Guide",
+		ID:          "apps_intro",
+		IsHidden:    true,
 		CascadeType: true,
 	},
 }