From 7f4e93785cd49471b4d902620c9ea4d523d874c7 Mon Sep 17 00:00:00 2001 From: Pacien Date: Tue, 16 Jul 2013 12:50:25 +0200 Subject: First version --- README.md | 43 ++++++++++- common.go | 143 ------------------------------------- compiled.go | 43 ----------- context.go | 97 ------------------------- dynamic.go | 82 --------------------- interactive.go | 131 ---------------------------------- main.go | 221 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 7 files changed, 228 insertions(+), 532 deletions(-) delete mode 100644 common.go delete mode 100644 compiled.go delete mode 100644 context.go delete mode 100644 dynamic.go delete mode 100644 interactive.go diff --git a/README.md b/README.md index 1df51f5..6d737a0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,45 @@ FoldaWeb ======== -A tree structure based website generator. +### Description + +FoldaWeb is a "keep last legacy" website generator: the program generates a page from parts of pages that are presents inside a directory and directly inside its parents. If multiple parts have the same name, it uses the last one (the one located the deepest in the directories before the current included). + +This behaviour makes particularly easy to create well-organized websites with many subpages of different types with associated layouts for each. + +___ + +### Features + +- Unique "keep last legacy" generation (no pun intended) +- Mustache templating: FoldaWeb uses [Mustache](http://mustache.github.io/mustache.5.html) as template engine and adds several handy contextual variables +- Markdown compatible: pages can be written using the [Markdown syntax](http://daringfireball.net/projects/markdown/syntax) + +Moreover, because FoldaWeb generates static files, generated websites are: + +- **Portable**: any host and web server software can serve flat files. +- **Fast**: no server-side scripting is required everytime someone loads a page +- **Secure**: no CMS security flaws + +___ + +### Example + +[Multiverse Inc. Global Website](http://multiverse.pacien.net) is an example of website generated using FoldaWeb. + +Its sources are available on GitHub at [Pacien/FoldaWeb-example](https://github.com/Pacien/FoldaWeb-example) + +___ + +### Usage + +Simply put the binary inside a directory containing a `source` folder with the website's sources inside and run the program (simply open the executable). Another folder named `out` containing the generated website will be created instantly. + +You can also pass custom settings via command line arguments: + + -sourceDir="./source": Path to the source directory. + -outputDir="./out": Path to the output directory. + -parsableExts="html, txt, md": Parsable file extensions separated by commas. + -saveAs="index.html": Save compiled files as named. + -startWith="index": Name without extension of the first file that will by parsed. + -wordSeparator="-": Word separator used to replace spaces in URLs. diff --git a/common.go b/common.go deleted file mode 100644 index 3203945..0000000 --- a/common.go +++ /dev/null @@ -1,143 +0,0 @@ -/* - - This file is part of FoldaWeb - - FoldaWeb is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - FoldaWeb is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with FoldaWeb. If not, see . - -*/ - -package main - -import ( - "bytes" - "fmt" - "github.com/Pacien/fcmd" - "github.com/drbawb/mustache" - "github.com/russross/blackfriday" - "io/ioutil" - "path" - "strings" - "sync" -) - -var wait sync.WaitGroup - -// Common templating - -func isParsable(fileName string, exts []string) bool { - for _, ext := range exts { - if path.Ext(fileName) == ext { - return true - } - } - return false -} - -func read(fileName string) ([]byte, error) { - fileBody, err := ioutil.ReadFile(fileName) - if err != nil { - return nil, err - } - if path.Ext(fileName) == ".md" { - fileBody = blackfriday.MarkdownCommon(fileBody) - } - return fileBody, nil -} - -func merge(files map[string][]byte) (merged []byte) { - merged = files["index"] - for pass := 0; bytes.Contains(merged, []byte("{{> ")) && pass < 4000; pass++ { - for fileName, fileBody := range files { - merged = bytes.Replace(merged, []byte("{{> "+fileName+"}}"), fileBody, -1) - } - } - return -} - -// COMPILED and INTERACTIVE modes - -func parse(dirPath string, elements map[string][]byte, exts []string, overwrite bool) (map[string][]byte, bool) { - parsed := false - _, filesList := fcmd.Ls(dirPath) - for _, fileName := range filesList { - if isParsable(fileName, exts) && (overwrite || elements[fileName[:len(fileName)-len(path.Ext(fileName))]] == nil) { - var err error - elements[fileName[:len(fileName)-len(path.Ext(fileName))]], err = read(path.Join(dirPath, fileName)) - if err != nil { - fmt.Println(err) - } - parsed = true - } - } - return elements, parsed -} - -func compile(dirPath string, elements map[string][]byte, sourceDir, outputDir, saveAs string, exts []string, recursive bool) { - defer wait.Done() - - if strings.HasPrefix(dirPath, outputDir) { - return - } - - parsed := false - elements, parsed = parse(dirPath, elements, exts, true) - - if recursive { - dirs, _ := fcmd.Ls(dirPath) - for _, dir := range dirs { - wait.Add(1) - go compile(path.Join(dirPath, dir), elements, sourceDir, outputDir, saveAs, exts, recursive) - } - } - - if !parsed { - return - } - - pagePath := strings.TrimPrefix(dirPath, sourceDir) - - template := merge(elements) - page := mustache.Render(string(template), makeContext(pagePath, sourceDir, exts)) - - err := fcmd.WriteFile(path.Join(outputDir, pagePath, saveAs), []byte(page)) - if err != nil { - fmt.Println(err) - return - } -} - -func copyFiles(dirPath, sourceDir, outputDir string, exts []string, recursive bool) { - defer wait.Done() - - if strings.HasPrefix(dirPath, outputDir) { - return - } - - dirs, files := fcmd.Ls(dirPath) - for _, file := range files { - if !isParsable(file, exts) { - err := fcmd.Cp(path.Join(dirPath, file), path.Join(outputDir, strings.TrimPrefix(dirPath, sourceDir), file)) - if err != nil { - fmt.Println(err) - } - } - } - - if recursive { - for _, dir := range dirs { - wait.Add(1) - go copyFiles(path.Join(dirPath, dir), sourceDir, outputDir, exts, recursive) - } - } -} diff --git a/compiled.go b/compiled.go deleted file mode 100644 index 7a3bd67..0000000 --- a/compiled.go +++ /dev/null @@ -1,43 +0,0 @@ -/* - - This file is part of FoldaWeb - - FoldaWeb is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - FoldaWeb is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with FoldaWeb. If not, see . - -*/ - -package main - -import ( - "fmt" - "os" -) - -func compiled(sourceDir, outputDir string, exts []string, saveAs string) { - // remove previously compiled site - err := os.RemoveAll(outputDir) - if err != nil { - fmt.Println(err) - return - } - - // compile everything - wait.Add(2) - go compile(sourceDir, make(map[string][]byte), sourceDir, outputDir, saveAs, exts, true) - go copyFiles(sourceDir, sourceDir, outputDir, exts, true) - - // wait until all tasks are completed - wait.Wait() - fmt.Println("Compilation done.") -} diff --git a/context.go b/context.go deleted file mode 100644 index 27a8c4a..0000000 --- a/context.go +++ /dev/null @@ -1,97 +0,0 @@ -/* - - This file is part of FoldaWeb - - FoldaWeb is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - FoldaWeb is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with FoldaWeb. If not, see . - -*/ - -package main - -import ( - "github.com/Pacien/fcmd" - "path" - "strings" -) - -type page struct { - Title string - Path string -} - -type context struct { - filePath string - Path string - IsCurrent func(params []string, data string) string - IsParent func(params []string, data string) string -} - -// Methods accessible in templates - -func (c context) Title() string { - _, t := path.Split(strings.TrimRight(c.Path, "/")) - return t -} - -func (c context) SubPages() (subPages []page) { - dirs, _ := fcmd.Ls(c.filePath) - for _, dir := range dirs { - var page page - page.Title = dir - page.Path = path.Join(c.Path, dir) - subPages = append(subPages, page) - } - return -} - -func (c context) IsRoot() bool { - if c.Path == "/" { - return true - } - return false -} - -func (c context) isCurrent(pageTitle string) bool { - if c.Title() == pageTitle { - return true - } - return false -} - -func (c context) isParent(pageTitle string) bool { - for _, parent := range strings.Split(c.Path, "/") { - if parent == pageTitle { - return true - } - } - return false -} - -func makeContext(pagePath, sourceDir string, exts []string) (c context) { - c.Path = path.Clean("/" + pagePath) - c.filePath = path.Join(sourceDir, c.Path) - c.IsCurrent = func(params []string, data string) string { - if c.isCurrent(strings.Join(params, " ")) { - return data - } - return "" - } - c.IsParent = func(params []string, data string) string { - if c.isParent(strings.Join(params, " ")) { - return data - } - return "" - } - return -} diff --git a/dynamic.go b/dynamic.go deleted file mode 100644 index 892ba71..0000000 --- a/dynamic.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - - This file is part of FoldaWeb - - FoldaWeb is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - FoldaWeb is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with FoldaWeb. If not, see . - -*/ - -package main - -import ( - "fmt" - "github.com/drbawb/mustache" - "net/http" - "path" - "strings" -) - -func handle(w http.ResponseWriter, r *http.Request) { - // serve static files - if !(path.Ext(r.URL.Path) == "" || isParsable(path.Ext(r.URL.Path), settings.exts)) { - http.ServeFile(w, r, path.Join(*settings.sourceDir, r.URL.Path)) - return - } - - // redirect to add the trailing slash if missing - if !strings.HasSuffix(r.URL.Path, "/") { - http.Redirect(w, r, r.URL.Path+"/", http.StatusFound) - return - } - - // get the list of dirs to parse - request := strings.TrimSuffix(r.URL.Path, "/") - dirs := strings.Split(request, "/") - for i, dir := range dirs { - if i != 0 { - dirs[i] = path.Join(dirs[i-1], dir) - } - } - - // parse these dirs - elements := make(map[string][]byte) - for i := len(dirs) - 1; i >= 0; i-- { - parsed := false - elements, parsed = parse(path.Join(*settings.sourceDir, dirs[i]), elements, settings.exts, false) - if (i == len(dirs)-1) && !parsed { - http.Error(w, "404 page not found", http.StatusNotFound) - return - } - } - - // render the page - template := merge(elements) - page := mustache.Render(string(template), makeContext(r.URL.Path, *settings.sourceDir, settings.exts)) - - // serve the page - _, err := w.Write([]byte(page)) - if err != nil { - fmt.Println(err) - return - } -} - -func dynamic(port string) { - fmt.Println("Listening on: localhost:" + port) - http.HandleFunc("/", handle) - err := http.ListenAndServe(":"+port, nil) - if err != nil { - fmt.Println(err) - } -} diff --git a/interactive.go b/interactive.go deleted file mode 100644 index 8ce32ca..0000000 --- a/interactive.go +++ /dev/null @@ -1,131 +0,0 @@ -/* - - This file is part of FoldaWeb - - FoldaWeb is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - FoldaWeb is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with FoldaWeb. If not, see . - -*/ - -package main - -import ( - "fmt" - "github.com/Pacien/fcmd" - "github.com/howeyc/fsnotify" - "os" - "path" - "strings" -) - -func watch(dirPath string, watcher *fsnotify.Watcher) *fsnotify.Watcher { - watcher.Watch(dirPath) - dirs, _ := fcmd.Explore(dirPath) - for _, dir := range dirs { - if !strings.HasPrefix(dir, *settings.outputDir) { - err := watcher.Watch(dir) - if err != nil { - fmt.Println(err) - } - } - } - return watcher -} - -func parseParents(dir, sourceDir string, exts []string) map[string][]byte { - dirs := strings.Split(strings.TrimPrefix(dir, sourceDir), "/") - elements := make(map[string][]byte) - for _, dir := range dirs { - elements, _ = parse(path.Join(sourceDir, dir), elements, exts, false) - } - return elements -} - -func interactive(sourceDir, outputDir string, exts []string, saveAs string) { - - // compile the whole site - compiled(sourceDir, outputDir, exts, saveAs) - - // watch the source dir - watcher, err := fsnotify.NewWatcher() - if err != nil { - fmt.Println(err) - } - defer watcher.Close() - watcher = watch(sourceDir, watcher) - - for { - select { - case ev := <-watcher.Event: - fmt.Println(ev) - - // ignore hidden files - if fcmd.IsHidden(ev.Name) { - break - } - - // manage watchers - if ev.IsDelete() || ev.IsRename() { - err = watcher.RemoveWatch(ev.Name) - if err != nil { - fmt.Println(err) - return - } - } else if ev.IsCreate() && fcmd.IsDir(ev.Name) { - watcher = watch(ev.Name, watcher) - } - - dir, _ := path.Split(ev.Name) - - // remove previously compiled files - if ev.IsDelete() || ev.IsRename() || ev.IsModify() { - var err error - if fcmd.IsDir(ev.Name) || !isParsable(ev.Name, exts) { - err = os.RemoveAll(path.Join(outputDir, strings.TrimPrefix(ev.Name, sourceDir))) - } else { - err = os.RemoveAll(path.Join(outputDir, strings.TrimPrefix(dir, sourceDir))) - } - if err != nil { - fmt.Println(err) - return - } - } - - // recompile changed files - if ev.IsCreate() || ev.IsModify() { - if fcmd.IsDir(ev.Name) { - elements := parseParents(ev.Name, sourceDir, exts) - dirPath := path.Join(sourceDir, strings.TrimPrefix(ev.Name, sourceDir)) - wait.Add(2) - go compile(dirPath, elements, sourceDir, outputDir, saveAs, exts, true) - go copyFiles(dirPath, sourceDir, outputDir, exts, true) - } else { - dirPath := path.Join(sourceDir, strings.TrimPrefix(dir, sourceDir)) - if isParsable(path.Ext(ev.Name), exts) { - elements := parseParents(dir, sourceDir, exts) - wait.Add(1) - go compile(dirPath, elements, sourceDir, outputDir, saveAs, exts, true) - } - wait.Add(1) - go copyFiles(dirPath, sourceDir, outputDir, exts, false) - } - } - - // wait until all tasks are completed - wait.Wait() - - case err := <-watcher.Error: - fmt.Println(err) - } - } -} diff --git a/main.go b/main.go index d93977c..050ffe6 100644 --- a/main.go +++ b/main.go @@ -17,58 +17,209 @@ */ +// FoldaWeb, a "keep last legacy" website generator package main import ( + "bytes" "flag" "fmt" "github.com/Pacien/fcmd" + "github.com/drbawb/mustache" + "github.com/frankbille/sanitize" + "github.com/russross/blackfriday" + "io/ioutil" + "path" "strings" + "sync" ) -var settings struct { - mode *string // compiled, interactive or dynamic - sourceDir *string - outputDir *string // for compiled site - port *string // for the integrated web server (dynamic mode only) - exts []string - saveAs *string +type generator struct { + // parameters + sourceDir, outputDir string + startWith, saveAs string + wordSeparator string + parsableExts []string + + // go routine sync + tasks sync.WaitGroup +} + +type page struct { + // properties accessible in templates + Title string + AbsPath, Path string + IsRoot bool + IsCurrent, IsParent func(params []string, data string) string + + // properties used for page generation + dirPath string + parts parts + body []byte } -func init() { - fcmd.DefaultPerm = 0755 // -rwxr-xr-x +type parts map[string][]byte + +// Creates an initiated generator +func newGenerator() (g generator) { + + // Read the command line arguments + flag.StringVar(&g.sourceDir, "sourceDir", "./source", "Path to the source directory.") + flag.StringVar(&g.outputDir, "outputDir", "./out", "Path to the output directory.") + flag.StringVar(&g.startWith, "startWith", "index", "Name without extension of the first file that will by parsed.") + flag.StringVar(&g.saveAs, "saveAs", "index.html", "Save compiled files as named.") + flag.StringVar(&g.wordSeparator, "wordSeparator", "-", "Word separator used to replace spaces in URLs.") + var parsableExts string + flag.StringVar(&parsableExts, "parsableExts", "html, txt, md", "Parsable file extensions separated by commas.") - // read settings - settings.mode = flag.String("mode", "compiled", "compiled|interactive|dynamic") - settings.sourceDir = flag.String("source", "./source", "Path to sources directory.") - settings.outputDir = flag.String("output", "./out", "[compiled mode] Path to output directory.") - settings.port = flag.String("port", "8080", "[dynamic mode] Port to listen.") - exts := flag.String("exts", "html, txt, md", "List parsable file extensions. Separated by commas.") - settings.saveAs = flag.String("saveAs", "index.html", "[compiled and interactive modes] Save compiled files as named.") flag.Parse() - settings.exts = strings.Split(*exts, ",") - for i, ext := range settings.exts { - settings.exts[i] = "." + strings.Trim(ext, ". ") + + g.sourceDir = path.Clean(g.sourceDir) + g.outputDir = path.Clean(g.outputDir) + for _, ext := range strings.Split(parsableExts, ",") { + g.parsableExts = append(g.parsableExts, "."+strings.Trim(ext, ". ")) + } + + return + +} + +func (g *generator) sanitizePath(filePath string) string { + sanitizedFilePath := strings.Replace(filePath, " ", g.wordSeparator, -1) + return sanitize.Path(sanitizedFilePath) +} + +func (g *generator) sourcePath(filePath string) string { + return path.Join(g.sourceDir, filePath) +} + +func (g *generator) outputPath(filePath string) string { + return path.Join(g.outputDir, g.sanitizePath(filePath)) +} + +func (g *generator) isFileParsable(fileName string) bool { + for _, ext := range g.parsableExts { + if path.Ext(fileName) == ext { + return true + } + } + return false +} + +func (g *generator) copyFile(filePath string) { + defer g.tasks.Done() + err := fcmd.Cp(g.sourcePath(filePath), g.outputPath(filePath)) + if err != nil { + fmt.Println(err) + } +} + +func (g *generator) parseFile(filePath string) []byte { + fileBody, err := ioutil.ReadFile(g.sourcePath(filePath)) + if err != nil { + fmt.Println(err) + return nil + } + if path.Ext(filePath) == ".md" { + fileBody = blackfriday.MarkdownCommon(fileBody) + } + return fileBody +} + +func (g *generator) mergeParts(parts parts) []byte { + merged := parts[g.startWith] + for pass := 0; bytes.Contains(merged, []byte("{{> ")) && pass < 1000; pass++ { + for partName, partBody := range parts { + merged = bytes.Replace(merged, []byte("{{> "+partName+"}}"), partBody, -1) + } + } + return merged +} + +func (g *generator) contextualize(page page) page { + _, page.Title = path.Split(page.dirPath) + if page.dirPath == "" { + page.IsRoot = true + page.AbsPath, page.Path = "/", "/" + } else { + page.AbsPath = g.sanitizePath("/" + page.dirPath) + _, page.Path = path.Split(page.AbsPath) + } + + page.IsCurrent = func(params []string, data string) string { + if page.Path == path.Clean(params[0]) { + return data + } + return "" + } + + page.IsParent = func(params []string, data string) string { + if strings.Contains(page.AbsPath, path.Clean(params[0])) { + return data + } + return "" + } + + return page +} + +func (g *generator) generate(page page) { + defer g.tasks.Done() + + dirs, files := fcmd.Ls(g.sourcePath(page.dirPath)) + + // Parse or copy files in the current directory + containsParsableFiles := false + for _, file := range files { + filePath := path.Join(page.dirPath, file) + if g.isFileParsable(file) { + containsParsableFiles = true + page.parts[file[:len(file)-len(path.Ext(file))]] = g.parseFile(filePath) + } else { + g.tasks.Add(1) + go g.copyFile(filePath) + } + } + + // Generate subpages in surdirectories + currentDirPath := page.dirPath + for _, dir := range dirs { + page.dirPath = path.Join(currentDirPath, dir) + g.tasks.Add(1) + go g.generate(page) + page.dirPath = currentDirPath + } + + // Generate the page at the current directory + if containsParsableFiles { + page.body = []byte(mustache.Render(string(g.mergeParts(page.parts)), g.contextualize(page))) + + err := fcmd.WriteFile(g.outputPath(path.Join(page.dirPath, g.saveAs)), page.body) + if err != nil { + fmt.Println(err) + } } - *settings.sourceDir = strings.TrimPrefix(*settings.sourceDir, "./") - *settings.outputDir = strings.TrimPrefix(*settings.outputDir, "./") } func main() { fmt.Println("FoldaWeb ") - fmt.Println("Mode: " + *settings.mode) - fmt.Println("Source: " + *settings.sourceDir) - fmt.Println("Output: " + *settings.outputDir) - fmt.Println("====================") - - switch *settings.mode { - case "compiled": - compiled(*settings.sourceDir, *settings.outputDir, settings.exts, *settings.saveAs) - case "interactive": - interactive(*settings.sourceDir, *settings.outputDir, settings.exts, *settings.saveAs) - case "dynamic": - dynamic(*settings.port) - default: - fmt.Println("Invalid mode.") + + g := newGenerator() + + // Remove previously generated site + err := fcmd.Rm(g.outputDir) + if err != nil { + fmt.Println(err) + return } + + // Generate everything + page := page{} + page.parts = make(parts) + g.tasks.Add(1) + go g.generate(page) + + // Wait until all tasks are completed + g.tasks.Wait() + fmt.Println("Done.") } -- cgit v1.2.3