Small refactor
This commit is contained in:
ViViDboarder
parent
e64ac5df22
commit
80d6c867cd
|
|
@ -9,3 +9,4 @@ docSet.dsidx
|
|||
docs.min.css
|
||||
holygrail.min.css
|
||||
syntax-highlighter.min.css
|
||||
*-version.txt
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"github.com/coopernurse/gorp"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
var dbmap *gorp.DbMap
|
||||
|
||||
func InitDb() *gorp.DbMap {
|
||||
db, err := sql.Open("sqlite3", "docSet.dsidx")
|
||||
ExitIfError(err)
|
||||
|
||||
dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}
|
||||
|
||||
dbmap.AddTableWithName(SearchIndex{}, "searchIndex").SetKeys(true, "ID")
|
||||
|
||||
err = dbmap.CreateTablesIfNotExists()
|
||||
ExitIfError(err)
|
||||
|
||||
err = dbmap.TruncateTables()
|
||||
ExitIfError(err)
|
||||
|
||||
return dbmap
|
||||
}
|
||||
|
||||
func SaveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
|
||||
if entry.LinkAttr.Href == "" || entryType == nil {
|
||||
return
|
||||
}
|
||||
|
||||
relLink := entry.GetContentFilepath(toc, false)
|
||||
name := entry.CleanTitle(*entryType)
|
||||
if entryType.ShowNamespace && len(entryHierarchy) > 0 {
|
||||
// Show namespace for methods
|
||||
name = entryHierarchy[len(entryHierarchy)-1] + "." + name
|
||||
}
|
||||
|
||||
// fmt.Println("Storing: " + name)
|
||||
|
||||
si := SearchIndex{
|
||||
Name: name,
|
||||
Type: entryType.TypeName,
|
||||
Path: relLink,
|
||||
}
|
||||
|
||||
dbmap.Insert(&si)
|
||||
}
|
||||
229
SFDashC/main.go
229
SFDashC/main.go
|
|
@ -1,12 +1,9 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/coopernurse/gorp"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
|
@ -16,88 +13,15 @@ import (
|
|||
"sync"
|
||||
)
|
||||
|
||||
/*
|
||||
TODO:
|
||||
- Move structs to own file
|
||||
- Move db stuff to own file
|
||||
- Stylesheets
|
||||
*/
|
||||
|
||||
// CSS Paths
|
||||
var cssBasePath = "https://developer.salesforce.com/resource/stylesheets"
|
||||
var cssFiles = []string{"holygrail.min.css", "docs.min.css", "syntax-highlighter.min.css"}
|
||||
|
||||
// JSON Structs
|
||||
|
||||
// AtlasTOC represents the meta documenation from Salesforce
|
||||
type AtlasTOC struct {
|
||||
AvailableVersions []VersionInfo `json:"available_versions"`
|
||||
Content string
|
||||
ContentDocumentID string `json:"content_document_id"`
|
||||
Deliverable string
|
||||
DocTitle string `json:"doc_title"`
|
||||
Locale string
|
||||
Language LanguageInfo
|
||||
PDFUrl string `json:"pdf_url"`
|
||||
TOCEntries []TOCEntry `json:"toc"`
|
||||
Title string
|
||||
Version VersionInfo
|
||||
}
|
||||
|
||||
// LanguageInfo contains information for linking and displaying the language
|
||||
type LanguageInfo struct {
|
||||
Label string
|
||||
Locale string
|
||||
URL string
|
||||
}
|
||||
|
||||
// VersionInfo representes a Salesforce documentation version
|
||||
type VersionInfo struct {
|
||||
DocVersion string `json:"doc_version"`
|
||||
ReleaseVersion string `json:"release_version"`
|
||||
VersionText string `json:"version_text"`
|
||||
VersionURL string `json:"version_url"`
|
||||
}
|
||||
|
||||
// TOCEntry represents a single Table of Contents item
|
||||
type TOCEntry struct {
|
||||
Text string
|
||||
ID string
|
||||
LinkAttr LinkAttr `json:"a_attr,omitempty"`
|
||||
Children []TOCEntry
|
||||
ComputedFirstTopic bool
|
||||
ComputedResetPageLayout bool
|
||||
}
|
||||
|
||||
// LinkAttr represents all attributes bound to a link
|
||||
type LinkAttr struct {
|
||||
Href string
|
||||
}
|
||||
|
||||
// TOCContent contains content information for a piece of documenation
|
||||
type TOCContent struct {
|
||||
ID string
|
||||
Title string
|
||||
Content string
|
||||
}
|
||||
|
||||
// Sqlite Struct
|
||||
|
||||
// SearchIndex is the database table that indexes the docs
|
||||
type SearchIndex struct {
|
||||
ID int64 `db:id`
|
||||
Name string `db:name`
|
||||
Type string `db:type`
|
||||
Path string `db:path`
|
||||
}
|
||||
|
||||
var dbmap *gorp.DbMap
|
||||
var wg sync.WaitGroup
|
||||
var throttle = make(chan int, maxConcurrency)
|
||||
|
||||
const maxConcurrency = 16
|
||||
|
||||
var throttle = make(chan int, maxConcurrency)
|
||||
|
||||
func parseFlags() (locale string, deliverables []string, silent bool) {
|
||||
flag.StringVar(
|
||||
&locale, "locale", "en-us",
|
||||
|
|
@ -172,6 +96,19 @@ func saveMainContent(toc *AtlasTOC) {
|
|||
}
|
||||
}
|
||||
|
||||
func saveContentVersion(toc *AtlasTOC) {
|
||||
filePath := fmt.Sprintf("%s-version.txt", toc.Deliverable)
|
||||
err := os.MkdirAll(filepath.Dir(filePath), 0755)
|
||||
ExitIfError(err)
|
||||
|
||||
ofile, err := os.Create(filePath)
|
||||
ExitIfError(err)
|
||||
|
||||
defer ofile.Close()
|
||||
_, err = ofile.WriteString(toc.Version.DocVersion)
|
||||
ExitIfError(err)
|
||||
}
|
||||
|
||||
func main() {
|
||||
locale, deliverables, silent := parseFlags()
|
||||
if silent {
|
||||
|
|
@ -186,19 +123,19 @@ func main() {
|
|||
}
|
||||
|
||||
// Init the Sqlite db
|
||||
dbmap = initDb()
|
||||
dbmap = InitDb()
|
||||
err := dbmap.TruncateTables()
|
||||
ExitIfError(err)
|
||||
|
||||
for _, deliverable := range deliverables {
|
||||
toc, err := getTOC(locale, deliverable)
|
||||
ExitIfError(err)
|
||||
|
||||
saveMainContent(toc)
|
||||
|
||||
err = verifyVersion(toc)
|
||||
WarnIfError(err)
|
||||
|
||||
saveMainContent(toc)
|
||||
saveContentVersion(toc)
|
||||
|
||||
// Download each entry
|
||||
for _, entry := range toc.TOCEntries {
|
||||
if entry.ID == "apex_reference" || entry.ID == "pages_compref" {
|
||||
|
|
@ -319,78 +256,6 @@ var supportedTypes = []SupportedType{
|
|||
},
|
||||
}
|
||||
|
||||
// IsType indicates that the TOCEntry is of a given SupportedType
|
||||
// This is done by checking the suffix of the entry text
|
||||
func (entry TOCEntry) IsType(t SupportedType) bool {
|
||||
return strings.HasSuffix(entry.Text, t.TitleSuffix)
|
||||
}
|
||||
|
||||
// CleanTitle trims known suffix from TOCEntry titles
|
||||
func (entry TOCEntry) CleanTitle(t SupportedType) string {
|
||||
if t.NoTrim {
|
||||
return entry.Text
|
||||
}
|
||||
return strings.TrimSuffix(entry.Text, " "+t.TitleSuffix)
|
||||
}
|
||||
|
||||
// GetRelLink extracts only the relative link from the Link Href
|
||||
func (entry TOCEntry) GetRelLink(removeAnchor bool) (relLink string) {
|
||||
if entry.LinkAttr.Href == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// Get the JSON file
|
||||
relLink = entry.LinkAttr.Href
|
||||
if removeAnchor {
|
||||
anchorIndex := strings.LastIndex(relLink, "#")
|
||||
if anchorIndex > 0 {
|
||||
relLink = relLink[0:anchorIndex]
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// GetContent retrieves Content for this TOCEntry from the API
|
||||
func (entry TOCEntry) GetContent(toc *AtlasTOC) (content *TOCContent, err error) {
|
||||
relLink := entry.GetRelLink(true)
|
||||
if relLink == "" {
|
||||
return
|
||||
}
|
||||
|
||||
url := fmt.Sprintf(
|
||||
"https://developer.salesforce.com/docs/get_document_content/%s/%s/%s/%s",
|
||||
toc.Deliverable,
|
||||
relLink,
|
||||
toc.Locale,
|
||||
toc.Version.DocVersion,
|
||||
)
|
||||
|
||||
// fmt.Println(url)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Read the downloaded JSON
|
||||
defer resp.Body.Close()
|
||||
contents, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Load into Struct
|
||||
content = new(TOCContent)
|
||||
err = json.Unmarshal([]byte(contents), content)
|
||||
if err != nil {
|
||||
fmt.Println("Error reading JSON")
|
||||
fmt.Println(resp.Status)
|
||||
fmt.Println(url)
|
||||
fmt.Println(string(contents))
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func getEntryType(entry TOCEntry) (*SupportedType, error) {
|
||||
if strings.HasPrefix(entry.ID, "pages_compref_") {
|
||||
return &SupportedType{
|
||||
|
|
@ -425,9 +290,9 @@ func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *Atlas
|
|||
|
||||
childType, err = getEntryType(child)
|
||||
if childType == nil && (entryType != nil && entryType.IsContainer) {
|
||||
saveSearchIndex(dbmap, child, entryType, toc)
|
||||
SaveSearchIndex(dbmap, child, entryType, toc)
|
||||
} else if childType != nil && !childType.IsContainer {
|
||||
saveSearchIndex(dbmap, child, childType, toc)
|
||||
SaveSearchIndex(dbmap, child, childType, toc)
|
||||
} else {
|
||||
WarnIfError(err)
|
||||
}
|
||||
|
|
@ -443,16 +308,6 @@ func processChildReferences(entry TOCEntry, entryType *SupportedType, toc *Atlas
|
|||
}
|
||||
}
|
||||
|
||||
// GetContentFilepath returns the filepath that should be used for the content
|
||||
func (entry TOCEntry) GetContentFilepath(toc *AtlasTOC, removeAnchor bool) string {
|
||||
relLink := entry.GetRelLink(removeAnchor)
|
||||
if relLink == "" {
|
||||
ExitIfError(NewFormatedError("Link not found for %s", entry.ID))
|
||||
}
|
||||
|
||||
return fmt.Sprintf("atlas.%s.%s.meta/%s/%s", toc.Locale, toc.Deliverable, toc.Deliverable, relLink)
|
||||
}
|
||||
|
||||
func downloadContent(entry TOCEntry, toc *AtlasTOC, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
|
||||
|
|
@ -508,47 +363,3 @@ func downloadCSS(fileName string, wg *sync.WaitGroup) {
|
|||
|
||||
<-throttle
|
||||
}
|
||||
|
||||
/**********************
|
||||
Database
|
||||
**********************/
|
||||
|
||||
func saveSearchIndex(dbmap *gorp.DbMap, entry TOCEntry, entryType *SupportedType, toc *AtlasTOC) {
|
||||
if entry.LinkAttr.Href == "" || entryType == nil {
|
||||
return
|
||||
}
|
||||
|
||||
relLink := entry.GetContentFilepath(toc, false)
|
||||
name := entry.CleanTitle(*entryType)
|
||||
if entryType.ShowNamespace && len(entryHierarchy) > 0 {
|
||||
// Show namespace for methods
|
||||
name = entryHierarchy[len(entryHierarchy)-1] + "." + name
|
||||
}
|
||||
|
||||
// fmt.Println("Storing: " + name)
|
||||
|
||||
si := SearchIndex{
|
||||
Name: name,
|
||||
Type: entryType.TypeName,
|
||||
Path: relLink,
|
||||
}
|
||||
|
||||
dbmap.Insert(&si)
|
||||
}
|
||||
|
||||
func initDb() *gorp.DbMap {
|
||||
db, err := sql.Open("sqlite3", "docSet.dsidx")
|
||||
ExitIfError(err)
|
||||
|
||||
dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}
|
||||
|
||||
dbmap.AddTableWithName(SearchIndex{}, "searchIndex").SetKeys(true, "ID")
|
||||
|
||||
err = dbmap.CreateTablesIfNotExists()
|
||||
ExitIfError(err)
|
||||
|
||||
err = dbmap.TruncateTables()
|
||||
ExitIfError(err)
|
||||
|
||||
return dbmap
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,155 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// JSON Structs
|
||||
|
||||
// AtlasTOC represents the meta documenation from Salesforce
|
||||
type AtlasTOC struct {
|
||||
AvailableVersions []VersionInfo `json:"available_versions"`
|
||||
Content string
|
||||
ContentDocumentID string `json:"content_document_id"`
|
||||
Deliverable string
|
||||
DocTitle string `json:"doc_title"`
|
||||
Locale string
|
||||
Language LanguageInfo
|
||||
PDFUrl string `json:"pdf_url"`
|
||||
TOCEntries []TOCEntry `json:"toc"`
|
||||
Title string
|
||||
Version VersionInfo
|
||||
}
|
||||
|
||||
// LanguageInfo contains information for linking and displaying the language
|
||||
type LanguageInfo struct {
|
||||
Label string
|
||||
Locale string
|
||||
URL string
|
||||
}
|
||||
|
||||
// VersionInfo representes a Salesforce documentation version
|
||||
type VersionInfo struct {
|
||||
DocVersion string `json:"doc_version"`
|
||||
ReleaseVersion string `json:"release_version"`
|
||||
VersionText string `json:"version_text"`
|
||||
VersionURL string `json:"version_url"`
|
||||
}
|
||||
|
||||
// TOCEntry represents a single Table of Contents item
|
||||
type TOCEntry struct {
|
||||
Text string
|
||||
ID string
|
||||
LinkAttr LinkAttr `json:"a_attr,omitempty"`
|
||||
Children []TOCEntry
|
||||
ComputedFirstTopic bool
|
||||
ComputedResetPageLayout bool
|
||||
}
|
||||
|
||||
// LinkAttr represents all attributes bound to a link
|
||||
type LinkAttr struct {
|
||||
Href string
|
||||
}
|
||||
|
||||
// TOCContent contains content information for a piece of documenation
|
||||
type TOCContent struct {
|
||||
ID string
|
||||
Title string
|
||||
Content string
|
||||
}
|
||||
|
||||
// Sqlite Struct
|
||||
|
||||
// SearchIndex is the database table that indexes the docs
|
||||
type SearchIndex struct {
|
||||
ID int64 `db:id`
|
||||
Name string `db:name`
|
||||
Type string `db:type`
|
||||
Path string `db:path`
|
||||
}
|
||||
|
||||
// IsType indicates that the TOCEntry is of a given SupportedType
|
||||
// This is done by checking the suffix of the entry text
|
||||
func (entry TOCEntry) IsType(t SupportedType) bool {
|
||||
return strings.HasSuffix(entry.Text, t.TitleSuffix)
|
||||
}
|
||||
|
||||
// CleanTitle trims known suffix from TOCEntry titles
|
||||
func (entry TOCEntry) CleanTitle(t SupportedType) string {
|
||||
if t.NoTrim {
|
||||
return entry.Text
|
||||
}
|
||||
return strings.TrimSuffix(entry.Text, " "+t.TitleSuffix)
|
||||
}
|
||||
|
||||
// GetRelLink extracts only the relative link from the Link Href
|
||||
func (entry TOCEntry) GetRelLink(removeAnchor bool) (relLink string) {
|
||||
if entry.LinkAttr.Href == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// Get the JSON file
|
||||
relLink = entry.LinkAttr.Href
|
||||
if removeAnchor {
|
||||
anchorIndex := strings.LastIndex(relLink, "#")
|
||||
if anchorIndex > 0 {
|
||||
relLink = relLink[0:anchorIndex]
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// GetContent retrieves Content for this TOCEntry from the API
|
||||
func (entry TOCEntry) GetContent(toc *AtlasTOC) (content *TOCContent, err error) {
|
||||
relLink := entry.GetRelLink(true)
|
||||
if relLink == "" {
|
||||
return
|
||||
}
|
||||
|
||||
url := fmt.Sprintf(
|
||||
"https://developer.salesforce.com/docs/get_document_content/%s/%s/%s/%s",
|
||||
toc.Deliverable,
|
||||
relLink,
|
||||
toc.Locale,
|
||||
toc.Version.DocVersion,
|
||||
)
|
||||
|
||||
// fmt.Println(url)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Read the downloaded JSON
|
||||
defer resp.Body.Close()
|
||||
contents, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Load into Struct
|
||||
content = new(TOCContent)
|
||||
err = json.Unmarshal([]byte(contents), content)
|
||||
if err != nil {
|
||||
fmt.Println("Error reading JSON")
|
||||
fmt.Println(resp.Status)
|
||||
fmt.Println(url)
|
||||
fmt.Println(string(contents))
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// GetContentFilepath returns the filepath that should be used for the content
|
||||
func (entry TOCEntry) GetContentFilepath(toc *AtlasTOC, removeAnchor bool) string {
|
||||
relLink := entry.GetRelLink(removeAnchor)
|
||||
if relLink == "" {
|
||||
ExitIfError(NewFormatedError("Link not found for %s", entry.ID))
|
||||
}
|
||||
|
||||
return fmt.Sprintf("atlas.%s.%s.meta/%s/%s", toc.Locale, toc.Deliverable, toc.Deliverable, relLink)
|
||||
}
|
||||
Loading…
Reference in New Issue