build based on df3f904
This commit is contained in:
zeptodoctor
parent
320d957658
commit
dbf8164336
|
|
@ -0,0 +1,260 @@
|
|||
// Generated by Documenter.jl
|
||||
requirejs.config({
|
||||
paths: {
|
||||
'highlight-julia': 'https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.15.10/languages/julia.min',
|
||||
'headroom': 'https://cdnjs.cloudflare.com/ajax/libs/headroom/0.10.3/headroom.min',
|
||||
'jqueryui': 'https://cdnjs.cloudflare.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min',
|
||||
'katex-auto-render': 'https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.11.1/contrib/auto-render.min',
|
||||
'jquery': 'https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min',
|
||||
'headroom-jquery': 'https://cdnjs.cloudflare.com/ajax/libs/headroom/0.10.3/jQuery.headroom.min',
|
||||
'katex': 'https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.11.1/katex.min',
|
||||
'highlight': 'https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.15.10/highlight.min',
|
||||
'highlight-julia-repl': 'https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.15.10/languages/julia-repl.min',
|
||||
},
|
||||
shim: {
|
||||
"highlight-julia": {
|
||||
"deps": [
|
||||
"highlight"
|
||||
]
|
||||
},
|
||||
"katex-auto-render": {
|
||||
"deps": [
|
||||
"katex"
|
||||
]
|
||||
},
|
||||
"headroom-jquery": {
|
||||
"deps": [
|
||||
"jquery",
|
||||
"headroom"
|
||||
]
|
||||
},
|
||||
"highlight-julia-repl": {
|
||||
"deps": [
|
||||
"highlight"
|
||||
]
|
||||
}
|
||||
}
|
||||
});
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
require(['jquery', 'katex', 'katex-auto-render'], function($, katex, renderMathInElement) {
|
||||
$(document).ready(function() {
|
||||
renderMathInElement(
|
||||
document.body,
|
||||
{
|
||||
"delimiters": [
|
||||
{
|
||||
"left": "$",
|
||||
"right": "$",
|
||||
"display": false
|
||||
},
|
||||
{
|
||||
"left": "$$",
|
||||
"right": "$$",
|
||||
"display": true
|
||||
},
|
||||
{
|
||||
"left": "\\[",
|
||||
"right": "\\]",
|
||||
"display": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
);
|
||||
})
|
||||
|
||||
})
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
require(['jquery', 'highlight', 'highlight-julia', 'highlight-julia-repl'], function($, hljs) {
|
||||
$(document).ready(function() {
|
||||
hljs.initHighlighting();
|
||||
})
|
||||
|
||||
})
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
require(['jquery', 'headroom', 'headroom-jquery'], function($, Headroom) {
|
||||
|
||||
// Manages the top navigation bar (hides it when the user starts scrolling down on the
|
||||
// mobile).
|
||||
window.Headroom = Headroom; // work around buggy module loading?
|
||||
$(document).ready(function() {
|
||||
$('#documenter .docs-navbar').headroom({
|
||||
"tolerance": {"up": 10, "down": 10},
|
||||
});
|
||||
})
|
||||
|
||||
})
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
require(['jquery'], function($) {
|
||||
|
||||
// Modal settings dialog
|
||||
$(document).ready(function() {
|
||||
var settings = $('#documenter-settings');
|
||||
$('#documenter-settings-button').click(function(){
|
||||
settings.toggleClass('is-active');
|
||||
});
|
||||
// Close the dialog if X is clicked
|
||||
$('#documenter-settings button.delete').click(function(){
|
||||
settings.removeClass('is-active');
|
||||
});
|
||||
// Close dialog if ESC is pressed
|
||||
$(document).keyup(function(e) {
|
||||
if (e.keyCode == 27) settings.removeClass('is-active');
|
||||
});
|
||||
});
|
||||
|
||||
})
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
require(['jquery'], function($) {
|
||||
|
||||
// Manages the showing and hiding of the sidebar.
|
||||
$(document).ready(function() {
|
||||
var sidebar = $("#documenter > .docs-sidebar");
|
||||
var sidebar_button = $("#documenter-sidebar-button")
|
||||
sidebar_button.click(function(ev) {
|
||||
ev.preventDefault();
|
||||
sidebar.toggleClass('visible');
|
||||
if (sidebar.hasClass('visible')) {
|
||||
// Makes sure that the current menu item is visible in the sidebar.
|
||||
$("#documenter .docs-menu a.is-active").focus();
|
||||
}
|
||||
});
|
||||
$("#documenter > .docs-main").bind('click', function(ev) {
|
||||
if ($(ev.target).is(sidebar_button)) {
|
||||
return;
|
||||
}
|
||||
if (sidebar.hasClass('visible')) {
|
||||
sidebar.removeClass('visible');
|
||||
}
|
||||
});
|
||||
})
|
||||
|
||||
// Resizes the package name / sitename in the sidebar if it is too wide.
|
||||
// Inspired by: https://github.com/davatron5000/FitText.js
|
||||
$(document).ready(function() {
|
||||
e = $("#documenter .docs-autofit");
|
||||
function resize() {
|
||||
var L = parseInt(e.css('max-width'), 10);
|
||||
var L0 = e.width();
|
||||
if(L0 > L) {
|
||||
var h0 = parseInt(e.css('font-size'), 10);
|
||||
e.css('font-size', L * h0 / L0);
|
||||
// TODO: make sure it survives resizes?
|
||||
}
|
||||
}
|
||||
// call once and then register events
|
||||
resize();
|
||||
$(window).resize(resize);
|
||||
$(window).on('orientationchange', resize);
|
||||
});
|
||||
|
||||
// Scroll the navigation bar to the currently selected menu item
|
||||
$(document).ready(function() {
|
||||
var sidebar = $("#documenter .docs-menu").get(0);
|
||||
var active = $("#documenter .docs-menu .is-active").get(0);
|
||||
if(typeof active !== 'undefined') {
|
||||
sidebar.scrollTop = active.offsetTop - sidebar.offsetTop - 15;
|
||||
}
|
||||
})
|
||||
|
||||
})
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
require(['jquery'], function($) {
|
||||
|
||||
function set_theme(theme) {
|
||||
var active = null;
|
||||
var disabled = [];
|
||||
for (var i = 0; i < document.styleSheets.length; i++) {
|
||||
var ss = document.styleSheets[i];
|
||||
var themename = ss.ownerNode.getAttribute("data-theme-name");
|
||||
if(themename === null) continue; // ignore non-theme stylesheets
|
||||
// Find the active theme
|
||||
if(themename === theme) active = ss;
|
||||
else disabled.push(ss);
|
||||
}
|
||||
if(active !== null) {
|
||||
active.disabled = false;
|
||||
if(active.ownerNode.getAttribute("data-theme-primary") === null) {
|
||||
document.getElementsByTagName('html')[0].className = "theme--" + theme;
|
||||
} else {
|
||||
document.getElementsByTagName('html')[0].className = "";
|
||||
}
|
||||
disabled.forEach(function(ss){
|
||||
ss.disabled = true;
|
||||
});
|
||||
}
|
||||
|
||||
// Store the theme in localStorage
|
||||
if(typeof(window.localStorage) !== "undefined") {
|
||||
window.localStorage.setItem("documenter-theme", theme);
|
||||
} else {
|
||||
console.error("Browser does not support window.localStorage");
|
||||
}
|
||||
}
|
||||
|
||||
// Theme picker setup
|
||||
$(document).ready(function() {
|
||||
// onchange callback
|
||||
$('#documenter-themepicker').change(function themepick_callback(ev){
|
||||
var themename = $('#documenter-themepicker option:selected').attr('value');
|
||||
set_theme(themename);
|
||||
});
|
||||
|
||||
// Make sure that the themepicker displays the correct theme when the theme is retrieved
|
||||
// from localStorage
|
||||
if(typeof(window.localStorage) !== "undefined") {
|
||||
var theme = window.localStorage.getItem("documenter-theme");
|
||||
if(theme !== null) {
|
||||
$('#documenter-themepicker option').each(function(i,e) {
|
||||
e.selected = (e.value === theme);
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
})
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
require(['jquery'], function($) {
|
||||
|
||||
// update the version selector with info from the siteinfo.js and ../versions.js files
|
||||
$(document).ready(function() {
|
||||
var version_selector = $("#documenter .docs-version-selector");
|
||||
var version_selector_select = $("#documenter .docs-version-selector select");
|
||||
|
||||
version_selector_select.change(function(x) {
|
||||
target_href = version_selector_select.children("option:selected").get(0).value;
|
||||
window.location.href = target_href;
|
||||
});
|
||||
|
||||
// add the current version to the selector based on siteinfo.js, but only if the selector is empty
|
||||
if (typeof DOCUMENTER_CURRENT_VERSION !== 'undefined' && $('#version-selector > option').length == 0) {
|
||||
var option = $("<option value='#' selected='selected'>" + DOCUMENTER_CURRENT_VERSION + "</option>");
|
||||
version_selector_select.append(option);
|
||||
}
|
||||
|
||||
if (typeof DOC_VERSIONS !== 'undefined') {
|
||||
var existing_versions = version_selector_select.children("option");
|
||||
var existing_versions_texts = existing_versions.map(function(i,x){return x.text});
|
||||
DOC_VERSIONS.forEach(function(each) {
|
||||
var version_url = documenterBaseURL + "/../" + each;
|
||||
var existing_id = $.inArray(each, existing_versions_texts);
|
||||
// if not already in the version selector, add it as a new option,
|
||||
// otherwise update the old option with the URL and enable it
|
||||
if (existing_id == -1) {
|
||||
var option = $("<option value='" + version_url + "'>" + each + "</option>");
|
||||
version_selector_select.append(option);
|
||||
} else {
|
||||
var option = existing_versions[existing_id];
|
||||
option.value = version_url;
|
||||
option.disabled = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// only show the version selector if the selector has been populated
|
||||
if (version_selector_select.children("option").length > 0) {
|
||||
version_selector.toggleClass("visible");
|
||||
}
|
||||
})
|
||||
|
||||
})
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
@import url('https://fonts.googleapis.com/css?family=Lato:400,400i');
|
||||
|
||||
body {
|
||||
font-family: Lato, "Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;
|
||||
}
|
||||
|
||||
nav.toc {
|
||||
padding-top: 0;
|
||||
background: rgb(240, 240, 240);
|
||||
line-height: 2em;
|
||||
cursor: default;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
h1+h2 {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
/* Green banner in ToC */
|
||||
nav.toc > h1 {
|
||||
margin-top: 0;
|
||||
padding-top: 0.4em;
|
||||
padding-bottom: 0.5em;
|
||||
border-bottom: 5px solid white;
|
||||
box-shadow: 0px -2px 5px rgb(60,60,60);
|
||||
margin-bottom: 0.5em;
|
||||
background: rgb(60, 150, 60);
|
||||
|
||||
font-style: italic;
|
||||
font-weight: normal;
|
||||
font-size: 50pt;
|
||||
text-transform: lowercase;
|
||||
text-shadow: 2px 2px 5px rgba(0,0,0,0.2);
|
||||
color: white;
|
||||
}
|
||||
|
||||
/* Reduce ToC font size */
|
||||
.toctext {
|
||||
font-size: 10pt;
|
||||
}
|
||||
|
||||
/* Fade out non-clickable ToC headers */
|
||||
nav.toc ul span.toctext {
|
||||
color: rgb(180, 180, 180);
|
||||
}
|
||||
|
||||
nav.toc ul .toctext {
|
||||
color: rgb(100, 100, 100);
|
||||
}
|
||||
|
||||
nav.toc ul a.toctext:hover {
|
||||
color: inherit;
|
||||
background: rgb(220, 220, 220);
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
nav.toc li.current > .toctext {
|
||||
background: linear-gradient(90deg, rgb(245,245,245) 0%, white 90%);
|
||||
font-weight: normal;
|
||||
}
|
||||
|
||||
nav.toc ul.internal li.toplevel {
|
||||
font-weight: normal;
|
||||
}
|
||||
|
||||
/* Content */
|
||||
|
||||
article { max-width: none; }
|
||||
|
||||
article > p, article > ul {
|
||||
max-width: 45em;
|
||||
}
|
||||
|
||||
/* Links */
|
||||
a, a:visited { color: rgb(0, 120, 0); }
|
||||
article p a { border-bottom: 1px solid rgb(200, 230, 200); }
|
||||
a:hover, a:visited:hover { color: rgb(0, 80, 0); }
|
||||
|
||||
/* Article Links */
|
||||
article p a { border-bottom: 1px solid rgb(200, 230, 200); }
|
||||
article p a:hover, article a:visited:hover { color: rgb(0, 120, 0); }
|
||||
article p a:hover { border-bottom: 1px solid rgb(150, 200, 150); }
|
||||
|
||||
/* Doctstrings */
|
||||
article section.docstring {
|
||||
padding: 0.5em 0;
|
||||
border-left: none;
|
||||
border-right: none;
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
/* Code */
|
||||
|
||||
article pre, article p > code {
|
||||
background: rgb(245, 250, 245);
|
||||
}
|
||||
|
||||
article pre {
|
||||
border: none;
|
||||
max-width: none;
|
||||
padding: 1em;
|
||||
border-radius: 10px 0px 0px 10px;
|
||||
margin-left: -1em;
|
||||
margin-right: -2em;
|
||||
}
|
||||
|
||||
.hljs-comment {
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.hljs-number {
|
||||
color: rgb(0, 150, 150);
|
||||
}
|
||||
|
|
@ -0,0 +1,248 @@
|
|||
// Generated by Documenter.jl
|
||||
requirejs.config({
|
||||
paths: {
|
||||
'lunr': 'https://cdnjs.cloudflare.com/ajax/libs/lunr.js/2.3.6/lunr.min',
|
||||
'lodash': 'https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.15/lodash.min',
|
||||
'jquery': 'https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min',
|
||||
}
|
||||
});
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
require(['jquery', 'lunr', 'lodash'], function($, lunr, _) {
|
||||
|
||||
$(document).ready(function() {
|
||||
// parseUri 1.2.2
|
||||
// (c) Steven Levithan <stevenlevithan.com>
|
||||
// MIT License
|
||||
function parseUri (str) {
|
||||
var o = parseUri.options,
|
||||
m = o.parser[o.strictMode ? "strict" : "loose"].exec(str),
|
||||
uri = {},
|
||||
i = 14;
|
||||
|
||||
while (i--) uri[o.key[i]] = m[i] || "";
|
||||
|
||||
uri[o.q.name] = {};
|
||||
uri[o.key[12]].replace(o.q.parser, function ($0, $1, $2) {
|
||||
if ($1) uri[o.q.name][$1] = $2;
|
||||
});
|
||||
|
||||
return uri;
|
||||
};
|
||||
parseUri.options = {
|
||||
strictMode: false,
|
||||
key: ["source","protocol","authority","userInfo","user","password","host","port","relative","path","directory","file","query","anchor"],
|
||||
q: {
|
||||
name: "queryKey",
|
||||
parser: /(?:^|&)([^&=]*)=?([^&]*)/g
|
||||
},
|
||||
parser: {
|
||||
strict: /^(?:([^:\/?#]+):)?(?:\/\/((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?))?((((?:[^?#\/]*\/)*)([^?#]*))(?:\?([^#]*))?(?:#(.*))?)/,
|
||||
loose: /^(?:(?![^:@]+:[^:@\/]*@)([^:\/?#.]+):)?(?:\/\/)?((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?)(((\/(?:[^?#](?![^?#\/]*\.[^?#\/.]+(?:[?#]|$)))*\/?)?([^?#\/]*))(?:\?([^#]*))?(?:#(.*))?)/
|
||||
}
|
||||
};
|
||||
|
||||
$("#search-form").submit(function(e) {
|
||||
e.preventDefault()
|
||||
})
|
||||
|
||||
// list below is the lunr 2.1.3 list minus the intersect with names(Base)
|
||||
// (all, any, get, in, is, which) and (do, else, for, let, where, while, with)
|
||||
// ideally we'd just filter the original list but it's not available as a variable
|
||||
lunr.stopWordFilter = lunr.generateStopWordFilter([
|
||||
'a',
|
||||
'able',
|
||||
'about',
|
||||
'across',
|
||||
'after',
|
||||
'almost',
|
||||
'also',
|
||||
'am',
|
||||
'among',
|
||||
'an',
|
||||
'and',
|
||||
'are',
|
||||
'as',
|
||||
'at',
|
||||
'be',
|
||||
'because',
|
||||
'been',
|
||||
'but',
|
||||
'by',
|
||||
'can',
|
||||
'cannot',
|
||||
'could',
|
||||
'dear',
|
||||
'did',
|
||||
'does',
|
||||
'either',
|
||||
'ever',
|
||||
'every',
|
||||
'from',
|
||||
'got',
|
||||
'had',
|
||||
'has',
|
||||
'have',
|
||||
'he',
|
||||
'her',
|
||||
'hers',
|
||||
'him',
|
||||
'his',
|
||||
'how',
|
||||
'however',
|
||||
'i',
|
||||
'if',
|
||||
'into',
|
||||
'it',
|
||||
'its',
|
||||
'just',
|
||||
'least',
|
||||
'like',
|
||||
'likely',
|
||||
'may',
|
||||
'me',
|
||||
'might',
|
||||
'most',
|
||||
'must',
|
||||
'my',
|
||||
'neither',
|
||||
'no',
|
||||
'nor',
|
||||
'not',
|
||||
'of',
|
||||
'off',
|
||||
'often',
|
||||
'on',
|
||||
'only',
|
||||
'or',
|
||||
'other',
|
||||
'our',
|
||||
'own',
|
||||
'rather',
|
||||
'said',
|
||||
'say',
|
||||
'says',
|
||||
'she',
|
||||
'should',
|
||||
'since',
|
||||
'so',
|
||||
'some',
|
||||
'than',
|
||||
'that',
|
||||
'the',
|
||||
'their',
|
||||
'them',
|
||||
'then',
|
||||
'there',
|
||||
'these',
|
||||
'they',
|
||||
'this',
|
||||
'tis',
|
||||
'to',
|
||||
'too',
|
||||
'twas',
|
||||
'us',
|
||||
'wants',
|
||||
'was',
|
||||
'we',
|
||||
'were',
|
||||
'what',
|
||||
'when',
|
||||
'who',
|
||||
'whom',
|
||||
'why',
|
||||
'will',
|
||||
'would',
|
||||
'yet',
|
||||
'you',
|
||||
'your'
|
||||
])
|
||||
|
||||
// add . as a separator, because otherwise "title": "Documenter.Anchors.add!"
|
||||
// would not find anything if searching for "add!", only for the entire qualification
|
||||
lunr.tokenizer.separator = /[\s\-\.]+/
|
||||
|
||||
// custom trimmer that doesn't strip @ and !, which are used in julia macro and function names
|
||||
lunr.trimmer = function (token) {
|
||||
return token.update(function (s) {
|
||||
return s.replace(/^[^a-zA-Z0-9@!]+/, '').replace(/[^a-zA-Z0-9@!]+$/, '')
|
||||
})
|
||||
}
|
||||
|
||||
lunr.Pipeline.registerFunction(lunr.stopWordFilter, 'juliaStopWordFilter')
|
||||
lunr.Pipeline.registerFunction(lunr.trimmer, 'juliaTrimmer')
|
||||
|
||||
var index = lunr(function () {
|
||||
this.ref('location')
|
||||
this.field('title',{boost: 100})
|
||||
this.field('text')
|
||||
documenterSearchIndex['docs'].forEach(function(e) {
|
||||
this.add(e)
|
||||
}, this)
|
||||
})
|
||||
var store = {}
|
||||
|
||||
documenterSearchIndex['docs'].forEach(function(e) {
|
||||
store[e.location] = {title: e.title, category: e.category}
|
||||
})
|
||||
|
||||
$(function(){
|
||||
searchresults = $('#documenter-search-results');
|
||||
searchinfo = $('#documenter-search-info');
|
||||
searchbox = $('#documenter-search-query');
|
||||
function update_search(querystring) {
|
||||
tokens = lunr.tokenizer(querystring)
|
||||
results = index.query(function (q) {
|
||||
tokens.forEach(function (t) {
|
||||
q.term(t.toString(), {
|
||||
fields: ["title"],
|
||||
boost: 100,
|
||||
usePipeline: false,
|
||||
editDistance: 0,
|
||||
wildcard: lunr.Query.wildcard.NONE
|
||||
})
|
||||
q.term(t.toString(), {
|
||||
fields: ["title"],
|
||||
boost: 10,
|
||||
usePipeline: false,
|
||||
editDistance: 2,
|
||||
wildcard: lunr.Query.wildcard.NONE
|
||||
})
|
||||
q.term(t.toString(), {
|
||||
fields: ["text"],
|
||||
boost: 1,
|
||||
usePipeline: true,
|
||||
editDistance: 0,
|
||||
wildcard: lunr.Query.wildcard.NONE
|
||||
})
|
||||
})
|
||||
})
|
||||
searchinfo.text("Number of results: " + results.length)
|
||||
searchresults.empty()
|
||||
results.forEach(function(result) {
|
||||
data = store[result.ref]
|
||||
link = $('<a class="docs-label">'+data.title+'</a>')
|
||||
link.attr('href', documenterBaseURL+'/'+result.ref)
|
||||
cat = $('<span class="docs-category">('+data.category+')</span>')
|
||||
li = $('<li>').append(link).append(" ").append(cat)
|
||||
searchresults.append(li)
|
||||
})
|
||||
}
|
||||
|
||||
function update_search_box() {
|
||||
querystring = searchbox.val()
|
||||
update_search(querystring)
|
||||
}
|
||||
|
||||
searchbox.keyup(_.debounce(update_search_box, 250))
|
||||
searchbox.change(update_search_box)
|
||||
|
||||
search_query_uri = parseUri(window.location).queryKey["q"]
|
||||
if(search_query_uri !== undefined) {
|
||||
search_query = decodeURIComponent(search_query_uri.replace(/\+/g, '%20'))
|
||||
searchbox.val(search_query)
|
||||
}
|
||||
update_search_box();
|
||||
})
|
||||
})
|
||||
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,42 @@
|
|||
// Small function to quickly swap out themes. Gets put into the <head> tag..
|
||||
function set_theme_from_local_storage() {
|
||||
// Browser does not support Web Storage, bail early.
|
||||
if(typeof(window.localStorage) === "undefined") return;
|
||||
// Get the user-picked theme from localStorage. May be `null`, which means the default
|
||||
// theme.
|
||||
var theme = window.localStorage.getItem("documenter-theme");
|
||||
// Initialize a few variables for the loop:
|
||||
//
|
||||
// - active: will contain the index of the theme that should be active. Note that there
|
||||
// is no guarantee that localStorage contains sane values. If `active` stays `null`
|
||||
// we either could not find the theme or it is the default (primary) theme anyway.
|
||||
// Either way, we then need to stick to the primary theme.
|
||||
//
|
||||
// - disabled: style sheets that should be disabled (i.e. all the theme style sheets
|
||||
// that are not the currently active theme)
|
||||
var active = null; var disabled = [];
|
||||
for (var i = 0; i < document.styleSheets.length; i++) {
|
||||
var ss = document.styleSheets[i];
|
||||
// The <link> tag of each style sheet is expected to have a data-theme-name attribute
|
||||
// which must contain the name of the theme. The names in localStorage much match this.
|
||||
var themename = ss.ownerNode.getAttribute("data-theme-name");
|
||||
// attribute not set => non-theme stylesheet => ignore
|
||||
if(themename === null) continue;
|
||||
// To distinguish the default (primary) theme, it needs to have the data-theme-primary
|
||||
// attribute set.
|
||||
var isprimary = (ss.ownerNode.getAttribute("data-theme-primary") !== null);
|
||||
// If we find a matching theme (and it's not the default), we'll set active to non-null
|
||||
if(!isprimary && themename === theme) active = i;
|
||||
// Store the style sheets of inactive themes so that we could disable them
|
||||
if(themename !== theme) disabled.push(ss);
|
||||
}
|
||||
if(active !== null) {
|
||||
// If we did find an active theme, we'll (1) add the theme--$(theme) class to <html>
|
||||
document.getElementsByTagName('html')[0].className = "theme--" + theme;
|
||||
// and (2) disable all the other theme stylesheets
|
||||
disabled.forEach(function(ss){
|
||||
ss.disabled = true;
|
||||
});
|
||||
}
|
||||
}
|
||||
set_theme_from_local_storage();
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,40 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><title>One-Hot Encoding · Flux</title><script>(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-36890222-9', 'auto');
|
||||
ga('send', 'pageview', {'page': location.pathname + location.search + location.hash});
|
||||
</script><link href="https://fonts.googleapis.com/css?family=Lato|Roboto+Mono" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/fontawesome.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/solid.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/brands.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.11.1/katex.min.css" rel="stylesheet" type="text/css"/><script>documenterBaseURL="../.."</script><script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js" data-main="../../assets/documenter.js"></script><script src="../../siteinfo.js"></script><script src="../../../versions.js"></script><link href="../../assets/flux.css" rel="stylesheet" type="text/css"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../../assets/themes/documenter-dark.css" data-theme-name="documenter-dark"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../../assets/themes/documenter-light.css" data-theme-name="documenter-light" data-theme-primary/><script src="../../assets/themeswap.js"></script></head><body><div id="documenter"><nav class="docs-sidebar"><div class="docs-package-name"><span class="docs-autofit">Flux</span></div><form class="docs-search" action="../../search/"><input class="docs-search-query" id="documenter-search-query" name="q" type="text" placeholder="Search docs"/></form><ul class="docs-menu"><li><a class="tocitem" href="../../">Home</a></li><li><span class="tocitem">Building Models</span><ul><li><a class="tocitem" href="../../models/basics/">Basics</a></li><li><a class="tocitem" href="../../models/recurrence/">Recurrence</a></li><li><a class="tocitem" href="../../models/regularisation/">Regularisation</a></li><li><a class="tocitem" href="../../models/layers/">Model Reference</a></li><li><a class="tocitem" href="../../models/advanced/">Advanced Model Building</a></li><li><a class="tocitem" href="../../models/nnlib/">NNlib</a></li></ul></li><li><span class="tocitem">Handling Data</span><ul><li class="is-active"><a class="tocitem" href>One-Hot Encoding</a><ul class="internal"><li><a class="tocitem" href="#Batches-1"><span>Batches</span></a></li></ul></li><li><a class="tocitem" href="../dataloader/">DataLoader</a></li></ul></li><li><span class="tocitem">Training Models</span><ul><li><a class="tocitem" href="../../training/optimisers/">Optimisers</a></li><li><a class="tocitem" href="../../training/training/">Training</a></li></ul></li><li><a class="tocitem" href="../../gpu/">GPU Support</a></li><li><a class="tocitem" href="../../saving/">Saving & Loading</a></li><li><a class="tocitem" href="../../ecosystem/">The Julia Ecosystem</a></li><li><a class="tocitem" href="../../performance/">Performance Tips</a></li><li><a class="tocitem" href="../../community/">Community</a></li></ul><div class="docs-version-selector field has-addons"><div class="control"><span class="docs-label button is-static is-size-7">Version</span></div><div class="docs-selector control is-expanded"><div class="select is-fullwidth is-size-7"><select id="documenter-version-selector"></select></div></div></div></nav><div class="docs-main"><header class="docs-navbar"><nav class="breadcrumb"><ul class="is-hidden-mobile"><li><a class="is-disabled">Handling Data</a></li><li class="is-active"><a href>One-Hot Encoding</a></li></ul><ul class="is-hidden-tablet"><li class="is-active"><a href>One-Hot Encoding</a></li></ul></nav><div class="docs-right"><a class="docs-edit-link" href="https://github.com/FluxML/Flux.jl/blob/master/docs/src/data/onehot.md" title="Edit on GitHub"><span class="docs-icon fab"></span><span class="docs-label is-hidden-touch">Edit on GitHub</span></a><a class="docs-settings-button fas fa-cog" id="documenter-settings-button" href="#" title="Settings"></a><a class="docs-sidebar-button fa fa-bars is-hidden-desktop" id="documenter-sidebar-button" href="#"></a></div></header><article class="content" id="documenter-page"><h1 id="One-Hot-Encoding-1"><a class="docs-heading-anchor" href="#One-Hot-Encoding-1">One-Hot Encoding</a><a class="docs-heading-anchor-permalink" href="#One-Hot-Encoding-1" title="Permalink"></a></h1><p>It's common to encode categorical variables (like <code>true</code>, <code>false</code> or <code>cat</code>, <code>dog</code>) in "one-of-k" or <a href="https://en.wikipedia.org/wiki/One-hot">"one-hot"</a> form. Flux provides the <code>onehot</code> function to make this easy.</p><pre><code class="language-none">julia> using Flux: onehot, onecold
|
||||
|
||||
julia> onehot(:b, [:a, :b, :c])
|
||||
3-element Flux.OneHotVector:
|
||||
false
|
||||
true
|
||||
false
|
||||
|
||||
julia> onehot(:c, [:a, :b, :c])
|
||||
3-element Flux.OneHotVector:
|
||||
false
|
||||
false
|
||||
true</code></pre><p>The inverse is <code>onecold</code> (which can take a general probability distribution, as well as just booleans).</p><pre><code class="language-julia">julia> onecold(ans, [:a, :b, :c])
|
||||
:c
|
||||
|
||||
julia> onecold([true, false, false], [:a, :b, :c])
|
||||
:a
|
||||
|
||||
julia> onecold([0.3, 0.2, 0.5], [:a, :b, :c])
|
||||
:c</code></pre><h2 id="Batches-1"><a class="docs-heading-anchor" href="#Batches-1">Batches</a><a class="docs-heading-anchor-permalink" href="#Batches-1" title="Permalink"></a></h2><p><code>onehotbatch</code> creates a batch (matrix) of one-hot vectors, and <code>onecold</code> treats matrices as batches.</p><pre><code class="language-julia">julia> using Flux: onehotbatch
|
||||
|
||||
julia> onehotbatch([:b, :a, :b], [:a, :b, :c])
|
||||
3×3 Flux.OneHotMatrix:
|
||||
false true false
|
||||
true false true
|
||||
false false false
|
||||
|
||||
julia> onecold(ans, [:a, :b, :c])
|
||||
3-element Array{Symbol,1}:
|
||||
:b
|
||||
:a
|
||||
:b</code></pre><p>Note that these operations returned <code>OneHotVector</code> and <code>OneHotMatrix</code> rather than <code>Array</code>s. <code>OneHotVector</code>s behave like normal vectors but avoid any unnecessary cost compared to using an integer index directly. For example, multiplying a matrix with a one-hot vector simply slices out the relevant row of the matrix under the hood.</p></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../../models/nnlib/">« NNlib</a><a class="docs-footer-nextpage" href="../dataloader/">DataLoader »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Wednesday 4 March 2020 05:16">Wednesday 4 March 2020</span>. Using Julia version 1.3.1.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,39 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><title>Regularisation · Flux</title><script>(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-36890222-9', 'auto');
|
||||
ga('send', 'pageview', {'page': location.pathname + location.search + location.hash});
|
||||
</script><link href="https://fonts.googleapis.com/css?family=Lato|Roboto+Mono" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/fontawesome.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/solid.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/brands.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.11.1/katex.min.css" rel="stylesheet" type="text/css"/><script>documenterBaseURL="../.."</script><script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js" data-main="../../assets/documenter.js"></script><script src="../../siteinfo.js"></script><script src="../../../versions.js"></script><link href="../../assets/flux.css" rel="stylesheet" type="text/css"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../../assets/themes/documenter-dark.css" data-theme-name="documenter-dark"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../../assets/themes/documenter-light.css" data-theme-name="documenter-light" data-theme-primary/><script src="../../assets/themeswap.js"></script></head><body><div id="documenter"><nav class="docs-sidebar"><div class="docs-package-name"><span class="docs-autofit">Flux</span></div><form class="docs-search" action="../../search/"><input class="docs-search-query" id="documenter-search-query" name="q" type="text" placeholder="Search docs"/></form><ul class="docs-menu"><li><a class="tocitem" href="../../">Home</a></li><li><span class="tocitem">Building Models</span><ul><li><a class="tocitem" href="../basics/">Basics</a></li><li><a class="tocitem" href="../recurrence/">Recurrence</a></li><li class="is-active"><a class="tocitem" href>Regularisation</a></li><li><a class="tocitem" href="../layers/">Model Reference</a></li><li><a class="tocitem" href="../advanced/">Advanced Model Building</a></li><li><a class="tocitem" href="../nnlib/">NNlib</a></li></ul></li><li><span class="tocitem">Handling Data</span><ul><li><a class="tocitem" href="../../data/onehot/">One-Hot Encoding</a></li><li><a class="tocitem" href="../../data/dataloader/">DataLoader</a></li></ul></li><li><span class="tocitem">Training Models</span><ul><li><a class="tocitem" href="../../training/optimisers/">Optimisers</a></li><li><a class="tocitem" href="../../training/training/">Training</a></li></ul></li><li><a class="tocitem" href="../../gpu/">GPU Support</a></li><li><a class="tocitem" href="../../saving/">Saving & Loading</a></li><li><a class="tocitem" href="../../ecosystem/">The Julia Ecosystem</a></li><li><a class="tocitem" href="../../performance/">Performance Tips</a></li><li><a class="tocitem" href="../../community/">Community</a></li></ul><div class="docs-version-selector field has-addons"><div class="control"><span class="docs-label button is-static is-size-7">Version</span></div><div class="docs-selector control is-expanded"><div class="select is-fullwidth is-size-7"><select id="documenter-version-selector"></select></div></div></div></nav><div class="docs-main"><header class="docs-navbar"><nav class="breadcrumb"><ul class="is-hidden-mobile"><li><a class="is-disabled">Building Models</a></li><li class="is-active"><a href>Regularisation</a></li></ul><ul class="is-hidden-tablet"><li class="is-active"><a href>Regularisation</a></li></ul></nav><div class="docs-right"><a class="docs-edit-link" href="https://github.com/FluxML/Flux.jl/blob/master/docs/src/models/regularisation.md" title="Edit on GitHub"><span class="docs-icon fab"></span><span class="docs-label is-hidden-touch">Edit on GitHub</span></a><a class="docs-settings-button fas fa-cog" id="documenter-settings-button" href="#" title="Settings"></a><a class="docs-sidebar-button fa fa-bars is-hidden-desktop" id="documenter-sidebar-button" href="#"></a></div></header><article class="content" id="documenter-page"><h1 id="Regularisation-1"><a class="docs-heading-anchor" href="#Regularisation-1">Regularisation</a><a class="docs-heading-anchor-permalink" href="#Regularisation-1" title="Permalink"></a></h1><p>Applying regularisation to model parameters is straightforward. We just need to apply an appropriate regulariser, such as <code>norm</code>, to each model parameter and add the result to the overall loss.</p><p>For example, say we have a simple regression.</p><pre><code class="language-julia">using Flux: crossentropy
|
||||
m = Dense(10, 5)
|
||||
loss(x, y) = crossentropy(softmax(m(x)), y)</code></pre><p>We can regularise this by taking the (L2) norm of the parameters, <code>m.W</code> and <code>m.b</code>.</p><pre><code class="language-julia">using LinearAlgebra
|
||||
|
||||
penalty() = norm(m.W) + norm(m.b)
|
||||
loss(x, y) = crossentropy(softmax(m(x)), y) + penalty()</code></pre><p>When working with layers, Flux provides the <code>params</code> function to grab all parameters at once. We can easily penalise everything with <code>sum(norm, params)</code>.</p><pre><code class="language-julia">julia> params(m)
|
||||
2-element Array{Any,1}:
|
||||
param([0.355408 0.533092; … 0.430459 0.171498])
|
||||
param([0.0, 0.0, 0.0, 0.0, 0.0])
|
||||
|
||||
julia> sum(norm, params(m))
|
||||
26.01749952921026</code></pre><p>Here's a larger example with a multi-layer perceptron.</p><pre><code class="language-julia">m = Chain(
|
||||
Dense(28^2, 128, relu),
|
||||
Dense(128, 32, relu),
|
||||
Dense(32, 10), softmax)
|
||||
|
||||
loss(x, y) = crossentropy(m(x), y) + sum(norm, params(m))
|
||||
|
||||
loss(rand(28^2), rand(10))</code></pre><p>One can also easily add per-layer regularisation via the <code>activations</code> function:</p><pre><code class="language-julia">julia> using Flux: activations
|
||||
|
||||
julia> c = Chain(Dense(10, 5, σ), Dense(5, 2), softmax)
|
||||
Chain(Dense(10, 5, σ), Dense(5, 2), softmax)
|
||||
|
||||
julia> activations(c, rand(10))
|
||||
3-element Array{Any,1}:
|
||||
Float32[0.84682214, 0.6704139, 0.42177814, 0.257832, 0.36255655]
|
||||
Float32[0.1501253, 0.073269576]
|
||||
Float32[0.5192045, 0.48079553]
|
||||
|
||||
julia> sum(norm, ans)
|
||||
2.1166067f0</code></pre></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../recurrence/">« Recurrence</a><a class="docs-footer-nextpage" href="../layers/">Model Reference »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Wednesday 4 March 2020 05:16">Wednesday 4 March 2020</span>. Using Julia version 1.3.1.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,50 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><title>Saving & Loading · Flux</title><script>(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-36890222-9', 'auto');
|
||||
ga('send', 'pageview', {'page': location.pathname + location.search + location.hash});
|
||||
</script><link href="https://fonts.googleapis.com/css?family=Lato|Roboto+Mono" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/fontawesome.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/solid.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/brands.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.11.1/katex.min.css" rel="stylesheet" type="text/css"/><script>documenterBaseURL=".."</script><script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js" data-main="../assets/documenter.js"></script><script src="../siteinfo.js"></script><script src="../../versions.js"></script><link href="../assets/flux.css" rel="stylesheet" type="text/css"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-dark.css" data-theme-name="documenter-dark"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-light.css" data-theme-name="documenter-light" data-theme-primary/><script src="../assets/themeswap.js"></script></head><body><div id="documenter"><nav class="docs-sidebar"><div class="docs-package-name"><span class="docs-autofit">Flux</span></div><form class="docs-search" action="../search/"><input class="docs-search-query" id="documenter-search-query" name="q" type="text" placeholder="Search docs"/></form><ul class="docs-menu"><li><a class="tocitem" href="../">Home</a></li><li><span class="tocitem">Building Models</span><ul><li><a class="tocitem" href="../models/basics/">Basics</a></li><li><a class="tocitem" href="../models/recurrence/">Recurrence</a></li><li><a class="tocitem" href="../models/regularisation/">Regularisation</a></li><li><a class="tocitem" href="../models/layers/">Model Reference</a></li><li><a class="tocitem" href="../models/advanced/">Advanced Model Building</a></li><li><a class="tocitem" href="../models/nnlib/">NNlib</a></li></ul></li><li><span class="tocitem">Handling Data</span><ul><li><a class="tocitem" href="../data/onehot/">One-Hot Encoding</a></li><li><a class="tocitem" href="../data/dataloader/">DataLoader</a></li></ul></li><li><span class="tocitem">Training Models</span><ul><li><a class="tocitem" href="../training/optimisers/">Optimisers</a></li><li><a class="tocitem" href="../training/training/">Training</a></li></ul></li><li><a class="tocitem" href="../gpu/">GPU Support</a></li><li class="is-active"><a class="tocitem" href>Saving & Loading</a><ul class="internal"><li><a class="tocitem" href="#Saving-Model-Weights-1"><span>Saving Model Weights</span></a></li><li><a class="tocitem" href="#Checkpointing-1"><span>Checkpointing</span></a></li></ul></li><li><a class="tocitem" href="../ecosystem/">The Julia Ecosystem</a></li><li><a class="tocitem" href="../performance/">Performance Tips</a></li><li><a class="tocitem" href="../community/">Community</a></li></ul><div class="docs-version-selector field has-addons"><div class="control"><span class="docs-label button is-static is-size-7">Version</span></div><div class="docs-selector control is-expanded"><div class="select is-fullwidth is-size-7"><select id="documenter-version-selector"></select></div></div></div></nav><div class="docs-main"><header class="docs-navbar"><nav class="breadcrumb"><ul class="is-hidden-mobile"><li class="is-active"><a href>Saving & Loading</a></li></ul><ul class="is-hidden-tablet"><li class="is-active"><a href>Saving & Loading</a></li></ul></nav><div class="docs-right"><a class="docs-edit-link" href="https://github.com/FluxML/Flux.jl/blob/master/docs/src/saving.md" title="Edit on GitHub"><span class="docs-icon fab"></span><span class="docs-label is-hidden-touch">Edit on GitHub</span></a><a class="docs-settings-button fas fa-cog" id="documenter-settings-button" href="#" title="Settings"></a><a class="docs-sidebar-button fa fa-bars is-hidden-desktop" id="documenter-sidebar-button" href="#"></a></div></header><article class="content" id="documenter-page"><h1 id="Saving-and-Loading-Models-1"><a class="docs-heading-anchor" href="#Saving-and-Loading-Models-1">Saving and Loading Models</a><a class="docs-heading-anchor-permalink" href="#Saving-and-Loading-Models-1" title="Permalink"></a></h1><p>You may wish to save models so that they can be loaded and run in a later session. The easiest way to do this is via <a href="https://github.com/MikeInnes/BSON.jl">BSON.jl</a>.</p><p>Save a model:</p><pre><code class="language-julia">julia> using Flux
|
||||
|
||||
julia> model = Chain(Dense(10,5,relu),Dense(5,2),softmax)
|
||||
Chain(Dense(10, 5, NNlib.relu), Dense(5, 2), NNlib.softmax)
|
||||
|
||||
julia> using BSON: @save
|
||||
|
||||
julia> @save "mymodel.bson" model</code></pre><p>Load it again:</p><pre><code class="language-julia">julia> using Flux
|
||||
|
||||
julia> using BSON: @load
|
||||
|
||||
julia> @load "mymodel.bson" model
|
||||
|
||||
julia> model
|
||||
Chain(Dense(10, 5, NNlib.relu), Dense(5, 2), NNlib.softmax)</code></pre><p>Models are just normal Julia structs, so it's fine to use any Julia storage format for this purpose. BSON.jl is particularly well supported and most likely to be forwards compatible (that is, models saved now will load in future versions of Flux).</p><div class="admonition is-info"><header class="admonition-header">Note</header><div class="admonition-body"><p>If a saved model's weights are stored on the GPU, the model will not load later on if there is no GPU support available. It's best to <a href="../gpu/">move your model to the CPU</a> with <code>cpu(model)</code> before saving it.</p></div></div><h2 id="Saving-Model-Weights-1"><a class="docs-heading-anchor" href="#Saving-Model-Weights-1">Saving Model Weights</a><a class="docs-heading-anchor-permalink" href="#Saving-Model-Weights-1" title="Permalink"></a></h2><p>In some cases it may be useful to save only the model parameters themselves, and rebuild the model architecture in your code. You can use <code>params(model)</code> to get model parameters. You can also use <code>data.(params)</code> to remove tracking.</p><pre><code class="language-Julia">julia> using Flux
|
||||
|
||||
julia> model = Chain(Dense(10,5,relu),Dense(5,2),softmax)
|
||||
Chain(Dense(10, 5, NNlib.relu), Dense(5, 2), NNlib.softmax)
|
||||
|
||||
julia> weights = params(model);
|
||||
|
||||
julia> using BSON: @save
|
||||
|
||||
julia> @save "mymodel.bson" weights</code></pre><p>You can easily load parameters back into a model with <code>Flux.loadparams!</code>.</p><pre><code class="language-julia">julia> using Flux
|
||||
|
||||
julia> model = Chain(Dense(10,5,relu),Dense(5,2),softmax)
|
||||
Chain(Dense(10, 5, NNlib.relu), Dense(5, 2), NNlib.softmax)
|
||||
|
||||
julia> using BSON: @load
|
||||
|
||||
julia> @load "mymodel.bson" weights
|
||||
|
||||
julia> Flux.loadparams!(model, weights)</code></pre><p>The new <code>model</code> we created will now be identical to the one we saved parameters for.</p><h2 id="Checkpointing-1"><a class="docs-heading-anchor" href="#Checkpointing-1">Checkpointing</a><a class="docs-heading-anchor-permalink" href="#Checkpointing-1" title="Permalink"></a></h2><p>In longer training runs it's a good idea to periodically save your model, so that you can resume if training is interrupted (for example, if there's a power cut). You can do this by saving the model in the <a href="../training/training/">callback provided to <code>train!</code></a>.</p><pre><code class="language-julia">using Flux: throttle
|
||||
using BSON: @save
|
||||
|
||||
m = Chain(Dense(10,5,relu),Dense(5,2),softmax)
|
||||
|
||||
evalcb = throttle(30) do
|
||||
# Show loss
|
||||
@save "model-checkpoint.bson" model
|
||||
end</code></pre><p>This will update the <code>"model-checkpoint.bson"</code> file every thirty seconds.</p><p>You can get more advanced by saving a series of models throughout training, for example</p><pre><code class="language-julia">@save "model-$(now()).bson" model</code></pre><p>will produce a series of models like <code>"model-2018-03-06T02:57:10.41.bson"</code>. You could also store the current test set loss, so that it's easy to (for example) revert to an older copy of the model if it starts to overfit.</p><pre><code class="language-julia">@save "model-$(now()).bson" model loss = testloss()</code></pre><p>You can even store optimiser state alongside the model, to resume training exactly where you left off.</p><pre><code class="language-julia">opt = ADAM()
|
||||
@save "model-$(now()).bson" model opt</code></pre></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../gpu/">« GPU Support</a><a class="docs-footer-nextpage" href="../ecosystem/">The Julia Ecosystem »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Wednesday 4 March 2020 05:16">Wednesday 4 March 2020</span>. Using Julia version 1.3.1.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><title>Search · Flux</title><script>(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-36890222-9', 'auto');
|
||||
ga('send', 'pageview', {'page': location.pathname + location.search + location.hash});
|
||||
</script><link href="https://fonts.googleapis.com/css?family=Lato|Roboto+Mono" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/fontawesome.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/solid.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/brands.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.11.1/katex.min.css" rel="stylesheet" type="text/css"/><script>documenterBaseURL=".."</script><script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js" data-main="../assets/documenter.js"></script><script src="../siteinfo.js"></script><script src="../../versions.js"></script><link href="../assets/flux.css" rel="stylesheet" type="text/css"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-dark.css" data-theme-name="documenter-dark"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-light.css" data-theme-name="documenter-light" data-theme-primary/><script src="../assets/themeswap.js"></script></head><body><div id="documenter"><nav class="docs-sidebar"><div class="docs-package-name"><span class="docs-autofit">Flux</span></div><form class="docs-search" action><input class="docs-search-query" id="documenter-search-query" name="q" type="text" placeholder="Search docs"/></form><ul class="docs-menu"><li><a class="tocitem" href="../">Home</a></li><li><span class="tocitem">Building Models</span><ul><li><a class="tocitem" href="../models/basics/">Basics</a></li><li><a class="tocitem" href="../models/recurrence/">Recurrence</a></li><li><a class="tocitem" href="../models/regularisation/">Regularisation</a></li><li><a class="tocitem" href="../models/layers/">Model Reference</a></li><li><a class="tocitem" href="../models/advanced/">Advanced Model Building</a></li><li><a class="tocitem" href="../models/nnlib/">NNlib</a></li></ul></li><li><span class="tocitem">Handling Data</span><ul><li><a class="tocitem" href="../data/onehot/">One-Hot Encoding</a></li><li><a class="tocitem" href="../data/dataloader/">DataLoader</a></li></ul></li><li><span class="tocitem">Training Models</span><ul><li><a class="tocitem" href="../training/optimisers/">Optimisers</a></li><li><a class="tocitem" href="../training/training/">Training</a></li></ul></li><li><a class="tocitem" href="../gpu/">GPU Support</a></li><li><a class="tocitem" href="../saving/">Saving & Loading</a></li><li><a class="tocitem" href="../ecosystem/">The Julia Ecosystem</a></li><li><a class="tocitem" href="../performance/">Performance Tips</a></li><li><a class="tocitem" href="../community/">Community</a></li></ul><div class="docs-version-selector field has-addons"><div class="control"><span class="docs-label button is-static is-size-7">Version</span></div><div class="docs-selector control is-expanded"><div class="select is-fullwidth is-size-7"><select id="documenter-version-selector"></select></div></div></div></nav><div class="docs-main"><header class="docs-navbar"><nav class="breadcrumb"><ul class="is-hidden-mobile"><li class="is-active"><a href>Search</a></li></ul><ul class="is-hidden-tablet"><li class="is-active"><a href>Search</a></li></ul></nav><div class="docs-right"><a class="docs-settings-button fas fa-cog" id="documenter-settings-button" href="#" title="Settings"></a><a class="docs-sidebar-button fa fa-bars is-hidden-desktop" id="documenter-sidebar-button" href="#"></a></div></header><article><p id="documenter-search-info">Loading search...</p><ul id="documenter-search-results"></ul></article></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Wednesday 4 March 2020 05:16">Wednesday 4 March 2020</span>. Using Julia version 1.3.1.</p></section><footer class="modal-card-foot"></footer></div></div></div></body><script src="../search_index.js"></script><script src="../assets/search.js"></script></html>
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1 @@
|
|||
var DOCUMENTER_CURRENT_VERSION = "v0.10.3";
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><title>Optimisers · Flux</title><script>(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-36890222-9', 'auto');
|
||||
ga('send', 'pageview', {'page': location.pathname + location.search + location.hash});
|
||||
</script><link href="https://fonts.googleapis.com/css?family=Lato|Roboto+Mono" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/fontawesome.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/solid.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/brands.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.11.1/katex.min.css" rel="stylesheet" type="text/css"/><script>documenterBaseURL="../.."</script><script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js" data-main="../../assets/documenter.js"></script><script src="../../siteinfo.js"></script><script src="../../../versions.js"></script><link href="../../assets/flux.css" rel="stylesheet" type="text/css"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../../assets/themes/documenter-dark.css" data-theme-name="documenter-dark"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../../assets/themes/documenter-light.css" data-theme-name="documenter-light" data-theme-primary/><script src="../../assets/themeswap.js"></script></head><body><div id="documenter"><nav class="docs-sidebar"><div class="docs-package-name"><span class="docs-autofit">Flux</span></div><form class="docs-search" action="../../search/"><input class="docs-search-query" id="documenter-search-query" name="q" type="text" placeholder="Search docs"/></form><ul class="docs-menu"><li><a class="tocitem" href="../../">Home</a></li><li><span class="tocitem">Building Models</span><ul><li><a class="tocitem" href="../../models/basics/">Basics</a></li><li><a class="tocitem" href="../../models/recurrence/">Recurrence</a></li><li><a class="tocitem" href="../../models/regularisation/">Regularisation</a></li><li><a class="tocitem" href="../../models/layers/">Model Reference</a></li><li><a class="tocitem" href="../../models/advanced/">Advanced Model Building</a></li><li><a class="tocitem" href="../../models/nnlib/">NNlib</a></li></ul></li><li><span class="tocitem">Handling Data</span><ul><li><a class="tocitem" href="../../data/onehot/">One-Hot Encoding</a></li><li><a class="tocitem" href="../../data/dataloader/">DataLoader</a></li></ul></li><li><span class="tocitem">Training Models</span><ul><li class="is-active"><a class="tocitem" href>Optimisers</a><ul class="internal"><li><a class="tocitem" href="#Optimiser-Reference-1"><span>Optimiser Reference</span></a></li><li><a class="tocitem" href="#Optimiser-Interface-1"><span>Optimiser Interface</span></a></li><li><a class="tocitem" href="#Composing-Optimisers-1"><span>Composing Optimisers</span></a></li><li><a class="tocitem" href="#Decays-1"><span>Decays</span></a></li></ul></li><li><a class="tocitem" href="../training/">Training</a></li></ul></li><li><a class="tocitem" href="../../gpu/">GPU Support</a></li><li><a class="tocitem" href="../../saving/">Saving & Loading</a></li><li><a class="tocitem" href="../../ecosystem/">The Julia Ecosystem</a></li><li><a class="tocitem" href="../../performance/">Performance Tips</a></li><li><a class="tocitem" href="../../community/">Community</a></li></ul><div class="docs-version-selector field has-addons"><div class="control"><span class="docs-label button is-static is-size-7">Version</span></div><div class="docs-selector control is-expanded"><div class="select is-fullwidth is-size-7"><select id="documenter-version-selector"></select></div></div></div></nav><div class="docs-main"><header class="docs-navbar"><nav class="breadcrumb"><ul class="is-hidden-mobile"><li><a class="is-disabled">Training Models</a></li><li class="is-active"><a href>Optimisers</a></li></ul><ul class="is-hidden-tablet"><li class="is-active"><a href>Optimisers</a></li></ul></nav><div class="docs-right"><a class="docs-edit-link" href="https://github.com/FluxML/Flux.jl/blob/master/docs/src/training/optimisers.md" title="Edit on GitHub"><span class="docs-icon fab"></span><span class="docs-label is-hidden-touch">Edit on GitHub</span></a><a class="docs-settings-button fas fa-cog" id="documenter-settings-button" href="#" title="Settings"></a><a class="docs-sidebar-button fa fa-bars is-hidden-desktop" id="documenter-sidebar-button" href="#"></a></div></header><article class="content" id="documenter-page"><h1 id="Optimisers-1"><a class="docs-heading-anchor" href="#Optimisers-1">Optimisers</a><a class="docs-heading-anchor-permalink" href="#Optimisers-1" title="Permalink"></a></h1><p>Consider a <a href="../../models/basics/">simple linear regression</a>. We create some dummy data, calculate a loss, and backpropagate to calculate gradients for the parameters <code>W</code> and <code>b</code>.</p><pre><code class="language-julia">using Flux
|
||||
|
||||
W = rand(2, 5)
|
||||
b = rand(2)
|
||||
|
||||
predict(x) = (W * x) .+ b
|
||||
loss(x, y) = sum((predict(x) .- y).^2)
|
||||
|
||||
x, y = rand(5), rand(2) # Dummy data
|
||||
l = loss(x, y) # ~ 3
|
||||
|
||||
θ = Params([W, b])
|
||||
grads = gradient(() -> loss(x, y), θ)</code></pre><p>We want to update each parameter, using the gradient, in order to improve (reduce) the loss. Here's one way to do that:</p><pre><code class="language-julia">using Flux.Optimise: update!
|
||||
|
||||
η = 0.1 # Learning Rate
|
||||
for p in (W, b)
|
||||
update!(p, -η * grads[p])
|
||||
end</code></pre><p>Running this will alter the parameters <code>W</code> and <code>b</code> and our loss should go down. Flux provides a more general way to do optimiser updates like this.</p><pre><code class="language-julia">opt = Descent(0.1) # Gradient descent with learning rate 0.1
|
||||
|
||||
for p in (W, b)
|
||||
update!(opt, p, grads[p])
|
||||
end</code></pre><p>An optimiser <code>update!</code> accepts a parameter and a gradient, and updates the parameter according to the chosen rule. We can also pass <code>opt</code> to our <a href="../training/">training loop</a>, which will update all parameters of the model in a loop. However, we can now easily replace <code>Descent</code> with a more advanced optimiser such as <code>ADAM</code>.</p><h2 id="Optimiser-Reference-1"><a class="docs-heading-anchor" href="#Optimiser-Reference-1">Optimiser Reference</a><a class="docs-heading-anchor-permalink" href="#Optimiser-Reference-1" title="Permalink"></a></h2><p>All optimisers return an object that, when passed to <code>train!</code>, will update the parameters passed to it.</p><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.update!" href="#Flux.Optimise.update!"><code>Flux.Optimise.update!</code></a> — <span class="docstring-category">Function</span></header><section><div><pre><code class="language-julia">update!(opt, p, g)
|
||||
update!(opt, ps::Params, gs)</code></pre><p>Perform an update step of the parameters <code>ps</code> (or the single parameter <code>p</code>) according to optimizer <code>opt</code> and the gradients <code>gs</code> (the gradient <code>g</code>).</p><p>As a result, the parameters are mutated and the optimizer's internal state may change. </p><p>update!(x, x̄)</p><p>Update the array <code>x</code> according to <code>x .-= x̄</code>.</p></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/train.jl#L5-L17">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.Descent" href="#Flux.Optimise.Descent"><code>Flux.Optimise.Descent</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">Descent(η)</code></pre><p>Classic gradient descent optimiser with learning rate <code>η</code>. For each parameter <code>p</code> and its gradient <code>δp</code>, this runs <code>p -= η*δp</code></p><p><strong>Parameters</strong></p><ul><li>Learning Rate (η): The amount by which the gradients are discounted before updating the weights. Defaults to <code>0.1</code>.</li></ul><p><strong>Example</strong></p><pre><code class="language-julia-repl">opt = Descent() # uses default η (0.1)
|
||||
|
||||
opt = Descent(0.3) # use provided η
|
||||
|
||||
ps = params(model)
|
||||
|
||||
gs = gradient(ps) do
|
||||
loss(x, y)
|
||||
end
|
||||
|
||||
Flux.Optimise.update!(opt, ps, gs)</code></pre></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L8-L31">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.Momentum" href="#Flux.Optimise.Momentum"><code>Flux.Optimise.Momentum</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">Momentum(η, ρ)</code></pre><p>Gradient descent with learning rate <code>η</code> and momentum <code>ρ</code>.</p><p><strong>Parameters</strong></p><ul><li>Learning Rate (<code>η</code>): Amount by which gradients are discounted before updating the weights. Defaults to <code>0.01</code>.</li><li>Momentum (<code>ρ</code>): Parameter that accelerates descent in the relevant direction and dampens oscillations. Defaults to <code>0.9</code>.</li></ul><p><strong>Examples</strong></p><pre><code class="language-julia">opt = Momentum() # uses defaults of η = 0.01 and ρ = 0.9
|
||||
|
||||
opt = Momentum(0.01, 0.99)</code></pre></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L42-L57">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.Nesterov" href="#Flux.Optimise.Nesterov"><code>Flux.Optimise.Nesterov</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">Nesterov(η, ρ)</code></pre><p>Gradient descent with learning rate <code>η</code> and Nesterov momentum <code>ρ</code>.</p><p><strong>Parameters</strong></p><ul><li>Learning Rate (η): Amount by which the gradients are dicsounted berfore updating the weights. Defaults to <code>0.001</code>.</li><li>Nesterov Momentum (ρ): Parameters controlling the amount of nesterov momentum to be applied. Defaults to <code>0.9</code>.</li></ul><p><strong>Examples</strong></p><pre><code class="language-julia">opt = Nesterov() # uses defaults η = 0.001 and ρ = 0.9
|
||||
|
||||
opt = Nesterov(0.003, 0.95)</code></pre></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L73-L88">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.RMSProp" href="#Flux.Optimise.RMSProp"><code>Flux.Optimise.RMSProp</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">RMSProp(η, ρ)</code></pre><p>Implements the RMSProp algortihm. Often a good choice for recurrent networks. Parameters other than learning rate generally don't need tuning.</p><p><strong>Parameters</strong></p><ul><li>Learning Rate (η): Defaults to <code>0.001</code>.</li><li>Rho (ρ): Defaults to <code>0.9</code>.</li></ul><p><strong>Examples</strong></p><pre><code class="language-julia">opt = RMSProp() # uses default η = 0.001 and ρ = 0.9
|
||||
|
||||
opt = RMSProp(0.002, 0.95)</code></pre><p><strong>References</strong></p><p><a href="https://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf">RMSProp</a></p></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L105-L123">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.ADAM" href="#Flux.Optimise.ADAM"><code>Flux.Optimise.ADAM</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">ADAM(η, β::Tuple)</code></pre><p>Implements the ADAM optimiser.</p><p><strong>Paramters</strong></p><ul><li>Learning Rate (<code>η</code>): Defaults to <code>0.001</code>.</li><li>Beta (<code>β::Tuple</code>): The first element refers to β1 and the second to β2. Defaults to <code>(0.9, 0.999)</code>.</li></ul><p><strong>Examples</strong></p><pre><code class="language-julia">opt = ADAM() # uses the default η = 0.001 and β = (0.9, 0.999)
|
||||
|
||||
opt = ADAM(0.001, (0.9, 0.8))</code></pre><p><strong>References</strong></p><p><a href="https://arxiv.org/abs/1412.6980v8">ADAM</a> optimiser.</p></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L139-L157">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.AdaMax" href="#Flux.Optimise.AdaMax"><code>Flux.Optimise.AdaMax</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">AdaMax(η, β::Tuple)</code></pre><p>Variant of ADAM based on ∞-norm.</p><p><strong>Parameters</strong></p><ul><li>Learning Rate (η): Defaults to <code>0.001</code></li><li>Beta (β::Tuple): The first element refers to β1 and the second to β2. Defaults to <code>(0.9, 0.999)</code>.</li></ul><p><strong>Examples</strong></p><pre><code class="language-julia">opt = AdaMax() # uses default η and β
|
||||
|
||||
opt = AdaMax(0.001, (0.9, 0.995))</code></pre><p><strong>References</strong></p><p><a href="https://arxiv.org/abs/1412.6980v9">AdaMax</a> optimiser.</p></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L221-L238">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.ADAGrad" href="#Flux.Optimise.ADAGrad"><code>Flux.Optimise.ADAGrad</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">ADAGrad(η)</code></pre><p>Implements AdaGrad. It has parameter specific learning rates based on how frequently it is updated.</p><p><strong>Parameters</strong></p><ul><li>Learning Rate (η): Defaults to <code>0.1</code></li></ul><p><strong>Examples</strong></p><pre><code class="language-julia">opt = ADAGrad() # uses default η = 0.1
|
||||
|
||||
opt = ADAGrad(0.001)</code></pre><p><strong>References</strong></p><p><a href="http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf">ADAGrad</a> optimiser. Parameters don't need tuning.</p></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L257-L275">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.ADADelta" href="#Flux.Optimise.ADADelta"><code>Flux.Optimise.ADADelta</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">ADADelta(ρ)</code></pre><p>Version of ADAGrad that adapts learning rate based on a window of past gradient updates. Parameters don't need tuning.</p><p><strong>Parameters</strong></p><ul><li>Rho (ρ): Factor by which gradient is decayed at each time step. Defaults to <code>0.9</code>.</li></ul><p><strong>Examples</strong></p><pre><code class="language-julia">opt = ADADelta() # uses default ρ = 0.9
|
||||
opt = ADADelta(0.89)</code></pre><p><strong>References</strong></p><p><a href="https://arxiv.org/abs/1212.5701">ADADelta</a> optimiser.</p></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L290-L306">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.AMSGrad" href="#Flux.Optimise.AMSGrad"><code>Flux.Optimise.AMSGrad</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">AMSGrad(η, β::Tuple)</code></pre><p>Implements AMSGrad version of the ADAM optimiser. Parameters don't need tuning.</p><p><strong>Parameters</strong></p><ul><li>Learning Rate (η): Defaults to <code>0.001</code>.</li><li>Beta (β::Tuple): The first element refers to β1 and the second to β2. Defaults to <code>(0.9, 0.999)</code>.</li></ul><p><strong>Examples</strong></p><pre><code class="language-julia">opt = AMSGrad() # uses default η and β
|
||||
opt = AMSGrad(0.001, (0.89, 0.995))</code></pre><p><strong>References</strong></p><p><a href="https://openreview.net/forum?id=ryQu7f-RZ">AMSGrad</a> optimiser.</p></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L323-L340">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.NADAM" href="#Flux.Optimise.NADAM"><code>Flux.Optimise.NADAM</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">NADAM(η, β::Tuple)</code></pre><p>Nesterov variant of ADAM. Parameters don't need tuning.</p><p><strong>Parameters</strong></p><ul><li>Learning Rate (η): Defaults to <code>0.001</code>.</li><li>Beta (β::Tuple): The first element refers to β1 and the second to β2. Defaults to <code>(0.9, 0.999)</code>.</li></ul><p><strong>Examples</strong></p><pre><code class="language-julia">opt = NADAM() # uses default η and β
|
||||
opt = NADAM(0.002, (0.89, 0.995))</code></pre><p><strong>References</strong></p><p><a href="http://cs229.stanford.edu/proj2015/054_report.pdf">NADAM</a> optimiser.</p></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L358-L375">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.ADAMW" href="#Flux.Optimise.ADAMW"><code>Flux.Optimise.ADAMW</code></a> — <span class="docstring-category">Function</span></header><section><div><pre><code class="language-julia">ADAMW(η, β::Tuple, decay)</code></pre><p>Variant of ADAM defined by fixing weight decay regularization.</p><p><strong>Parameters</strong></p><ul><li>Learning Rate (η): Defaults to <code>0.001</code>.</li><li>Beta (β::Tuple): The first element refers to β1 and the second to β2. Defaults to (0.9, 0.999).</li><li>decay: Decay applied to weights during optimisation. Defaults to 0.</li></ul><p><strong>Examples</strong></p><pre><code class="language-julia">opt = ADAMW() # uses default η, β and decay
|
||||
opt = ADAMW(0.001, (0.89, 0.995), 0.1)</code></pre><p><strong>References</strong></p><p><a href="https://arxiv.org/abs/1711.05101">ADAMW</a></p></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L394-L412">source</a></section></article><h2 id="Optimiser-Interface-1"><a class="docs-heading-anchor" href="#Optimiser-Interface-1">Optimiser Interface</a><a class="docs-heading-anchor-permalink" href="#Optimiser-Interface-1" title="Permalink"></a></h2><p>Flux's optimisers are built around a <code>struct</code> that holds all the optimiser parameters along with a definition of how to apply the update rule associated with it. We do this via the <code>apply!</code> function which takes the optimiser as the first argument followed by the parameter and its corresponding gradient.</p><p>In this manner Flux also allows one to create custom optimisers to be used seamlessly. Let's work this with a simple example.</p><pre><code class="language-julia">mutable struct Momentum
|
||||
eta
|
||||
rho
|
||||
velocity
|
||||
end
|
||||
|
||||
Momentum(eta::Real, rho::Real) = Momentum(eta, rho, IdDict())</code></pre><p>The <code>Momentum</code> type will act as our optimiser in this case. Notice that we have added all the parameters as fields, along with the velocity which we will use as our state dictionary. Each parameter in our models will get an entry in there. We can now define the rule applied when this optimiser is invoked.</p><pre><code class="language-julia">function apply!(o::Momentum, x, Δ)
|
||||
η, ρ = o.eta, o.rho
|
||||
v = get!(o.velocity, x, zero(x))::typeof(x)
|
||||
@. v = ρ * v - η * Δ
|
||||
@. Δ = -v
|
||||
end</code></pre><p>This is the basic definition of a Momentum update rule given by:</p><div>\[v = ρ * v - η * Δ
|
||||
w = w - v\]</div><p>The <code>apply!</code> defines the update rules for an optimiser <code>opt</code>, given the parameters and gradients. It returns the updated gradients. Here, every parameter <code>x</code> is retrieved from the running state <code>v</code> and subsequently updates the state of the optimiser.</p><p>Flux internally calls on this function via the <code>update!</code> function. It shares the API with <code>apply!</code> but ensures that multiple parameters are handled gracefully.</p><h2 id="Composing-Optimisers-1"><a class="docs-heading-anchor" href="#Composing-Optimisers-1">Composing Optimisers</a><a class="docs-heading-anchor-permalink" href="#Composing-Optimisers-1" title="Permalink"></a></h2><p>Flux defines a special kind of optimiser simply called <code>Optimiser</code> which takes in arbitrary optimisers as input. Its behaviour is similar to the usual optimisers, but differs in that it acts by calling the optimisers listed in it sequentially. Each optimiser produces a modified gradient that will be fed into the next, and the resultant update will be applied to the parameter as usual. A classic use case is where adding decays is desirable. Flux defines some basic decays including <code>ExpDecay</code>, <code>InvDecay</code> etc.</p><pre><code class="language-julia">opt = Optimiser(ExpDecay(0.001, 0.1, 1000, 1e-4), Descent())</code></pre><p>Here we apply exponential decay to the <code>Descent</code> optimiser. The defaults of <code>ExpDecay</code> say that its learning rate will be decayed every 1000 steps. It is then applied like any optimiser.</p><pre><code class="language-julia">w = randn(10, 10)
|
||||
w1 = randn(10,10)
|
||||
ps = Params([w, w1])
|
||||
|
||||
loss(x) = Flux.mse(w * x, w1 * x)
|
||||
|
||||
loss(rand(10)) # around 9
|
||||
|
||||
for t = 1:10^5
|
||||
θ = Params([w, w1])
|
||||
θ̄ = gradient(() -> loss(rand(10)), θ)
|
||||
Flux.Optimise.update!(opt, θ, θ̄)
|
||||
end
|
||||
|
||||
loss(rand(10)) # around 0.9</code></pre><p>In this manner it is possible to compose optimisers for some added flexibility.</p><h2 id="Decays-1"><a class="docs-heading-anchor" href="#Decays-1">Decays</a><a class="docs-heading-anchor-permalink" href="#Decays-1" title="Permalink"></a></h2><p>Similar to optimisers, Flux also defines some simple decays that can be used in conjunction with other optimisers, or standalone.</p><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.ExpDecay" href="#Flux.Optimise.ExpDecay"><code>Flux.Optimise.ExpDecay</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">ExpDecay(eta, decay, decay_step, clip)</code></pre><p>Discount the learning rate <code>eta</code> by a multiplicative factor <code>decay</code> every <code>decay_step</code> till a minimum of <code>clip</code>.</p><p><strong>Parameters</strong></p><ul><li>Learning Rate (eta): Defaults to <code>0.001</code>.</li><li>decay: Factor by which the learning rate is discounted. Defaults to <code>0.1</code>.</li><li>decay_step: Schedules decay operations by setting number of steps between two decay operations. Defaults to <code>1000</code>.</li><li>clip: Minimum value of learning rate. Defaults to <code>1e-4</code>.</li></ul><p><strong>Example</strong></p><p>To apply exponential decay to an optimiser:</p><pre><code class="language-julia">Optimiser(ExpDecay(..), Opt(..))
|
||||
opt = Optimiser(ExpDecay(), ADAM())</code></pre></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L471-L488">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.InvDecay" href="#Flux.Optimise.InvDecay"><code>Flux.Optimise.InvDecay</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">InvDecay(γ)</code></pre><p>Applies inverse time decay to an optimiser, i.e., the effective step size at iteration <code>n</code> is <code>eta / (1 + γ * n)</code> where <code>eta</code> is the initial step size. The wrapped optimiser's step size is not modified.</p><p><strong>Parameters</strong></p><ul><li>gamma (γ): Defaults to <code>0.001</code></li></ul><p><strong>Example</strong></p><pre><code class="language-julia">Optimiser(InvDecay(..), Opt(..))</code></pre></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L443-L455">source</a></section></article><article class="docstring"><header><a class="docstring-binding" id="Flux.Optimise.WeightDecay" href="#Flux.Optimise.WeightDecay"><code>Flux.Optimise.WeightDecay</code></a> — <span class="docstring-category">Type</span></header><section><div><pre><code class="language-julia">WeightDecay(wd)</code></pre><p>Decays the weight by <code>wd</code></p><p><strong>Parameters</strong></p><ul><li>weight decay (wd): 0</li></ul></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Flux.jl/blob/df3f904f7c34f095562693b6a9ca67047319dea0/src/optimise/optimisers.jl#L509-L516">source</a></section></article></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../../data/dataloader/">« DataLoader</a><a class="docs-footer-nextpage" href="../training/">Training »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Wednesday 4 March 2020 05:16">Wednesday 4 March 2020</span>. Using Julia version 1.3.1.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
|
||||
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue