Includes more guides and better logging

Large-ish refactor

Makefile

@@ -1,20 +1,22 @@
-default: complete
+.PHONY: all clean-index package-apex clean-index package-vf clean-index package-combined
 
-complete: clean-index run-apex package-apex clean-index run-vf package-vf
+default: all
 
-run-apex:
+all: clean-index package-apex clean-index package-vf clean-index package-combined
+
+run-apex: clean-index
 	dep ensure
-	(cd SFDashC && go run *.go --silent apexcode)
+	go run ./SFDashC/*.go apexcode
 
-run-vf:
+run-vf: clean-index
 	dep ensure
-	(cd SFDashC && go run *.go --silent pages)
+	go run ./SFDashC/*.go pages
 
-run-combined:
+run-combined: clean-index
 	dep ensure
-	(cd SFDashC && go run *.go --silent apexcode pages)
+	go run ./SFDashC/*.go apexcode pages
 
-package-apex:
+package-apex: run-apex
 	$(eval name = Apex)
 	$(eval package = Salesforce $(name).docset)
 	$(eval version = $(shell cat SFDashC/apexcode-version.txt))
@@ -26,7 +28,7 @@ package-apex:
 	cp SFDashC/docSet.dsidx "$(package)/Contents/Resources/"
 	@echo "Docset generated!"
 
-package-vf:
+package-vf: run-vf
 	$(eval name = Pages)
 	$(eval package = Salesforce $(name).docset)
 	$(eval version = $(shell cat SFDashC/pages-version.txt))
@@ -38,7 +40,7 @@ package-vf:
 	cp SFDashC/docSet.dsidx "$(package)/Contents/Resources/"
 	@echo "Docset generated!"
 
-package-combined:
+package-combined: run-combined
 	$(eval name = Combined)
 	$(eval package = Salesforce $(name).docset)
 	mkdir -p "$(package)/Contents/Resources/Documents"

SFDashC/database.go

@@ -37,8 +37,6 @@ func SaveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType
 		name = entryHierarchy[len(entryHierarchy)-1] + "." + name
 	}
 
-	// fmt.Println("Storing: " + name)
-
 	si := SearchIndex{
 		Name: name,
 		Type: entryType.TypeName,
@@ -46,4 +44,5 @@ func SaveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType
 	}
 
 	dbmap.Insert(&si)
+	LogDebug("%s is indexed as a %s", entry.Text, entryType.TypeName)
 }

SFDashC/errors.go

@@ -1,30 +1,14 @@
 package main
 
 import (
+	"errors"
 	"fmt"
-	"os"
+	"log"
 )
 
-var shouldWarn = true
-
-// Custom errors
-type errorString struct {
-	message string
-}
-
-// Error retrievies the Error message from the error
-func (err errorString) Error() string {
-	return err.message
-}
-
-// NoWarn disables all warning output
-func WithoutWarning() {
-	shouldWarn = false
-}
-
 // NewCustomError creates a custom error using a string as the message
 func NewCustomError(message string) error {
-	return &errorString{message}
+	return errors.New(message)
 }
 
 // NewFormatedError creates a new error using Sprintf
@@ -40,14 +24,13 @@ func NewTypeNotFoundError(entry TOCEntry) error {
 // ExitIfError is a helper function for terminating if an error is not nil
 func ExitIfError(err error) {
 	if err != nil {
-		fmt.Println("ERROR: ", err)
+		log.Fatal(err)
-		os.Exit(1)
 	}
 }
 
 // WarnIfError is a helper function for terminating if an error is not nil
 func WarnIfError(err error) {
-	if err != nil && shouldWarn {
+	if err != nil {
-		fmt.Println("WARNING: ", err)
+		LogDebug(err.Error())
 	}
 }

SFDashC/logging.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"fmt"
+	"log"
+)
+
+const (
+	prefix = "SFDashC"
+	// Log Levels
+	ERROR = iota
+	INFO  = iota
+	DEBUG = iota
+)
+
+var logLevel int
+
+func init() {
+	logLevel = INFO
+}
+
+func getLevelText() string {
+	switch logLevel {
+	case ERROR:
+		return "ERROR"
+	case INFO:
+		return "INFO"
+	case DEBUG:
+		return "DEBUG"
+	default:
+		return "UNKNOWN"
+	}
+}
+
+func getLogPrefix() string {
+	return fmt.Sprintf("%s: %s:", prefix, getLevelText())
+}
+
+// SetLogLevel will set the maximum level to print
+func SetLogLevel(level int) {
+	logLevel = level
+}
+
+// Log will print a formatted message with a prefix for a specified level
+// If the level is greater than the maximum log level, it will not print
+func Log(level int, format string, a ...interface{}) {
+	if level <= logLevel {
+		message := fmt.Sprintf(format, a...)
+		message = fmt.Sprintf("%s: %s: %s", prefix, getLevelText(), message)
+		log.Println(message)
+	}
+}
+
+// LogError will print an error message
+// If the level is greater than the maximum log level, it will not print
+// It is recommended to use log.Fatal() instead since it will handle exits for you
+func LogError(format string, a ...interface{}) {
+	Log(ERROR, format, a...)
+}
+
+// LogInfo will print an info message
+// If the level is greater than the maximum log level, it will not print
+func LogInfo(format string, a ...interface{}) {
+	Log(INFO, format, a...)
+}
+
+// LogDebug will print an debug message
+// If the level is greater than the maximum log level, it will not print
+func LogDebug(format string, a ...interface{}) {
+	Log(DEBUG, format, a...)
+}

SFDashC/main.go

@@ -9,7 +9,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"strings"
 	"sync"
 )
 
@@ -22,13 +21,13 @@ var throttle = make(chan int, maxConcurrency)
 
 const maxConcurrency = 16
 
-func parseFlags() (locale string, deliverables []string, silent bool) {
+func parseFlags() (locale string, deliverables []string, debug bool) {
 	flag.StringVar(
 		&locale, "locale", "en-us",
 		"locale to use for documentation (default: en-us)",
 	)
 	flag.BoolVar(
-		&silent, "silent", false, "this flag supresses warning messages",
+		&debug, "debug", false, "this flag supresses warning messages",
 	)
 	flag.Parse()
 
@@ -66,7 +65,7 @@ func verifyVersion(toc *AtlasTOC) error {
 }
 
 func printSuccess(toc *AtlasTOC) {
-	fmt.Println("Success:", toc.DocTitle, "-", toc.Version.VersionText, "-", toc.Version.DocVersion)
+	LogInfo("Success: %s - %s - %s", toc.DocTitle, toc.Version.VersionText, toc.Version.DocVersion)
 }
 
 func saveMainContent(toc *AtlasTOC) {
@@ -106,9 +105,9 @@ func saveContentVersion(toc *AtlasTOC) {
 }
 
 func main() {
-	locale, deliverables, silent := parseFlags()
+	locale, deliverables, debug := parseFlags()
-	if silent {
+	if debug {
-		WithoutWarning()
+		SetLogLevel(DEBUG)
 	}
 
 	// Download CSS
@@ -133,11 +132,15 @@ func main() {
 		saveContentVersion(toc)
 
 		// Download each entry
+		/*
+		 * topLevelEntryIDs := map[string]bool{
+		 * 	"apex_dev_guide": true,
+		 * 	"pages_compref":  true,
+		 * }
+		 */
 		for _, entry := range toc.TOCEntries {
-			if entry.ID == "apex_dev_guide" || entry.ID == "pages_compref" {
 			processChildReferences(entry, nil, toc)
 		}
-		}
 
 		printSuccess(toc)
 	}
@@ -145,121 +148,8 @@ func main() {
 	wg.Wait()
 }
 
-// SupportedType contains information for generating indexes for types we care about
-type SupportedType struct {
-	TypeName, TitleSuffix                                                     string
-	PushName, AppendParents, IsContainer, NoTrim, ShowNamespace, ParseContent bool
-}
-
-var supportedTypes = []SupportedType{
-	SupportedType{
-		TypeName:      "Method",
-		TitleSuffix:   "Methods",
-		AppendParents: true,
-		IsContainer:   true,
-		ShowNamespace: true,
-	},
-	SupportedType{
-		TypeName:      "Constructor",
-		TitleSuffix:   "Constructors",
-		AppendParents: true,
-		IsContainer:   true,
-		ShowNamespace: false,
-	},
-	SupportedType{
-		TypeName:      "Class",
-		TitleSuffix:   "Class",
-		PushName:      true,
-		AppendParents: true,
-		ShowNamespace: false,
-	},
-	SupportedType{
-		TypeName:      "Namespace",
-		TitleSuffix:   "Namespace",
-		PushName:      true,
-		AppendParents: true,
-		ShowNamespace: false,
-	},
-	SupportedType{
-		TypeName:      "Interface",
-		TitleSuffix:   "Interface",
-		PushName:      true,
-		AppendParents: true,
-		ShowNamespace: false,
-	},
-	SupportedType{
-		TypeName:      "Statement",
-		TitleSuffix:   "Statement",
-		ShowNamespace: false,
-	},
-	SupportedType{
-		TypeName:      "Enum",
-		TitleSuffix:   "Enum",
-		AppendParents: true,
-		ShowNamespace: false,
-	},
-	SupportedType{
-		TypeName:      "Property",
-		TitleSuffix:   "Properties",
-		AppendParents: true,
-		IsContainer:   true,
-		ShowNamespace: false,
-	},
-	SupportedType{
-		TypeName:      "Guide",
-		TitleSuffix:   "Example Implementation",
-		NoTrim:        true,
-		ShowNamespace: false,
-	},
-	SupportedType{
-		TypeName:      "Statement",
-		TitleSuffix:   "Statements",
-		NoTrim:        true,
-		AppendParents: false,
-		IsContainer:   true,
-		ShowNamespace: false,
-	},
-	SupportedType{
-		TypeName:      "Field",
-		TitleSuffix:   "Fields",
-		AppendParents: true,
-		PushName:      true,
-		IsContainer:   true,
-		ShowNamespace: false,
-	},
-	SupportedType{
-		TypeName:      "Exception",
-		TitleSuffix:   "Exceptions",
-		NoTrim:        true,
-		AppendParents: true,
-		ShowNamespace: false,
-		ParseContent:  true,
-	},
-	SupportedType{
-		TypeName:      "Constant",
-		TitleSuffix:   "Constants",
-		NoTrim:        true,
-		AppendParents: true,
-		ShowNamespace: false,
-		ParseContent:  true,
-	},
-	SupportedType{
-		TypeName:      "Class",
-		TitleSuffix:   "Class (Base Email Methods)",
-		PushName:      true,
-		AppendParents: true,
-		ShowNamespace: false,
-	},
-}
-
 func getEntryType(entry TOCEntry) (*SupportedType, error) {
-	if strings.HasPrefix(entry.ID, "pages_compref_") {
+	for _, t := range SupportedTypes {
-		return &SupportedType{
-			TypeName: "Tag",
-			NoTrim:   true,
-		}, nil
-	}
-	for _, t := range supportedTypes {
 		if entry.IsType(t) {
 			return &t, nil
 		}
@@ -275,7 +165,7 @@ func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *Atlas
 	}
 
 	for _, child := range entry.Children {
-		// fmt.Println("Processing: " + child.Text)
+		LogDebug("Processing: %s", child.Text)
 		var err error
 		var childType *SupportedType
 		if child.LinkAttr.Href != "" {
@@ -285,19 +175,24 @@ func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *Atlas
 			go downloadContent(child, toc, &wg)
 
 			childType, err = getEntryType(child)
-			if childType == nil && (entryType != nil && entryType.IsContainer) {
+			if childType == nil && entryType != nil && (entryType.IsContainer || entryType.CascadeType) {
-				SaveSearchIndex(dbmap, child, entryType, toc)
+				LogDebug("Parent was container or cascade, using parent type of %s", entryType.TypeName)
-			} else if childType != nil && !childType.IsContainer {
+				childType = entryType
+			}
+
+			if childType == nil {
+				WarnIfError(err)
+			} else if !childType.IsContainer {
 				SaveSearchIndex(dbmap, child, childType, toc)
 			} else {
-				WarnIfError(err)
+				LogDebug("%s is a container. Do not index", child.Text)
 			}
 		}
 		if len(child.Children) > 0 {
 			processChildReferences(child, childType, toc)
 		}
 	}
-	// fmt.Println("Done processing children for " + entry.Text)
+	LogDebug("Done processing children for %s", entry.Text)
 
 	if entryType != nil && entryType.PushName {
 		entryHierarchy = entryHierarchy[:len(entryHierarchy)-1]

SFDashC/structs.go

@@ -62,8 +62,37 @@ type TOCContent struct {
 	Content string
 }
 
-// Sqlite Struct
+// SupportedType contains information for generating indexes for types we care about
+type SupportedType struct {
+	// Exact match against an id
+	ID string
+	// Match against a prefix for the id
+	IDPrefix string
+	// Match against a prefix for the title
+	TitlePrefix string
+	// Match against a suffix for the title
+	TitleSuffix string
+	// Override Title
+	TitleOverride string
+	// Docset type
+	TypeName string
+	// Not sure...
+	AppendParents bool
+	// Indicates that this just contains other nodes and we don't want to index this
+	IsContainer bool
+	// Skip trimming of suffix from title
+	NoTrim bool
+	// Not sure...
+	ParseContent bool
+	// Should this name be pushed int othe path for child entries Eg. Class name prefix methods
+	PushName bool
+	// Should a namspace be prefixed to the database entry
+	ShowNamespace bool
+	// Should cascade type downwards
+	CascadeType bool
+}
 
+// Sqlite Struct
 // SearchIndex is the database table that indexes the docs
 type SearchIndex struct {
 	ID   int64  `db:id`
@@ -72,14 +101,38 @@ type SearchIndex struct {
 	Path string `db:path`
 }
 
+func (suppType SupportedType) matchesTitle(title string) bool {
+	match := false
+	if suppType.TitlePrefix != "" {
+		match = match || strings.HasPrefix(title, suppType.TitlePrefix)
+	}
+	if suppType.TitleSuffix != "" {
+		match = match || strings.HasSuffix(title, suppType.TitleSuffix)
+	}
+	return match
+}
+
+func (suppType SupportedType) matchesID(id string) bool {
+	if suppType.ID != "" && suppType.ID == id {
+		return true
+	}
+	if suppType.IDPrefix != "" {
+		return strings.HasPrefix(id, suppType.IDPrefix)
+	}
+	return false
+}
+
 // IsType indicates that the TOCEntry is of a given SupportedType
 // This is done by checking the suffix of the entry text
 func (entry TOCEntry) IsType(t SupportedType) bool {
-	return strings.HasSuffix(entry.Text, t.TitleSuffix)
+	return t.matchesTitle(entry.Text) || t.matchesID(entry.ID)
 }
 
 // CleanTitle trims known suffix from TOCEntry titles
 func (entry TOCEntry) CleanTitle(t SupportedType) string {
+	if t.TitleOverride != "" {
+		return t.TitleOverride
+	}
 	if t.NoTrim {
 		return entry.Text
 	}

SFDashC/supportedtypes.go

@@ -0,0 +1,178 @@
+package main
+
+var SupportedTypes = []SupportedType{
+	SupportedType{
+		TypeName:      "Method",
+		TitleSuffix:   "Methods",
+		AppendParents: true,
+		IsContainer:   true,
+		ShowNamespace: true,
+	},
+	SupportedType{
+		TypeName:      "Constructor",
+		TitleSuffix:   "Constructors",
+		AppendParents: true,
+		IsContainer:   true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Class",
+		TitleSuffix:   "Class",
+		PushName:      true,
+		AppendParents: true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Namespace",
+		TitleSuffix:   "Namespace",
+		PushName:      true,
+		AppendParents: true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Interface",
+		TitleSuffix:   "Interface",
+		PushName:      true,
+		AppendParents: true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Statement",
+		ID:            "langCon_apex_SOQL_query_all_rows",
+		TitleOverride: "ALL ROWS",
+	},
+	SupportedType{
+		TypeName:      "Statement",
+		TitleSuffix:   "Statement",
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Enum",
+		TitleSuffix:   "Enum",
+		AppendParents: true,
+		ShowNamespace: true,
+	},
+	SupportedType{
+		TypeName:      "Property",
+		TitleSuffix:   "Properties",
+		AppendParents: true,
+		IsContainer:   true,
+		ShowNamespace: true,
+	},
+	SupportedType{
+		TypeName:      "Guide",
+		TitleSuffix:   "Example Implementation",
+		NoTrim:        true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Statement",
+		TitleSuffix:   "Statements",
+		NoTrim:        true,
+		AppendParents: false,
+		IsContainer:   true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Field",
+		TitleSuffix:   "Fields",
+		AppendParents: true,
+		PushName:      true,
+		IsContainer:   true,
+		ShowNamespace: true,
+	},
+	SupportedType{
+		TypeName:      "Exception",
+		TitleSuffix:   "Exceptions",
+		NoTrim:        true,
+		AppendParents: true,
+		ShowNamespace: false,
+		ParseContent:  true,
+	},
+	SupportedType{
+		TypeName:      "Constant",
+		TitleSuffix:   "Constants",
+		NoTrim:        true,
+		AppendParents: true,
+		ShowNamespace: false,
+		ParseContent:  true,
+	},
+	SupportedType{
+		TypeName:      "Class",
+		TitleSuffix:   "Class (Base Email Methods)",
+		PushName:      true,
+		AppendParents: true,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		TypeName:      "Guide",
+		TitlePrefix:   "Best Practices",
+		TitleSuffix:   "Best Practices",
+		NoTrim:        true,
+		PushName:      false,
+		AppendParents: false,
+		ShowNamespace: false,
+	},
+	SupportedType{
+		IDPrefix: "pages_compref_",
+		TypeName: "Tag",
+		NoTrim:   true,
+	},
+	SupportedType{
+		TypeName: "Guide",
+		IDPrefix: "pages_maps",
+		NoTrim:   true,
+	},
+	SupportedType{
+		TypeName: "Guide",
+		IDPrefix: "pages_dynamic_vf",
+		NoTrim:   true,
+	},
+	SupportedType{
+		TypeName: "Guide",
+		IDPrefix: "pages_comp_cust",
+		NoTrim:   true,
+	},
+	SupportedType{
+		TypeName: "Guide",
+		IDPrefix: "pages_resources",
+		NoTrim:   true,
+	},
+	SupportedType{
+		TypeName: "Guide",
+		IDPrefix: "pages_controller",
+		NoTrim:   true,
+	},
+	SupportedType{
+		TypeName: "Guide",
+		IDPrefix: "pages_styling",
+		NoTrim:   true,
+	},
+	SupportedType{
+		TypeName: "Guide",
+		IDPrefix: "pages_security",
+		NoTrim:   true,
+	},
+	SupportedType{
+		TypeName:      "Variables",
+		TitleSuffix:   "Global Variables",
+		NoTrim:        true,
+		AppendParents: true,
+		ShowNamespace: false,
+		ParseContent:  true,
+		IsContainer:   true,
+	},
+	SupportedType{
+		TypeName: "Guide",
+		IDPrefix: "pages_variables_functions",
+	},
+	SupportedType{
+		TypeName: "Guide",
+		IDPrefix: "pages_variables_operators",
+	},
+	SupportedType{
+		TypeName:    "Guide",
+		ID:          "apex_intro_get_started",
+		CascadeType: true,
+	},
+}