Merge branch 'release/1.8.5'

.dockerignore

+*~
+.dockerignore
 .hg
+Dockerfile
 
 # Remove the git objects, logs, etc. to make final image smaller.
 # Some files still need to be in the .git directory, because Etherpad at

.github/ISSUE_TEMPLATE/bug_report.md

-* * *
-
+---
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: bug
-assignees: 
+labels: ''
+assignees: ''
 
-* * *
+---
 
 **Describe the bug**
 A clear and concise description of what the bug is.
@@ -24,23 +23,16 @@ A clear and concise description of what you expected to happen.
 **Screenshots**
 If applicable, add screenshots to help explain your problem.
 
-**Environment (please complete the following information):**
-
--   Etherpad Version: (e.g. 1.8.0)
--   Deployment (manual install, docker, ...)
-
 **Desktop (please complete the following information):**
-
--   OS: (e.g. iOS)
--   Browser (e.g. chrome, safari)
--   Version (e.g. 22)
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
 
 **Smartphone (please complete the following information):**
-
--   Device: (e.g. iPhone6)
--   OS: (e.g. iOS8.1)
--   Browser (e.g. stock browser, safari)
--   Version (e.g. 22)
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
 
 **Additional context**
 Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: Feature Request
+assignees: ''
+
+---
+
 * * *
 
 name: Feature request
@@ -19,3 +28,6 @@ A clear and concise description of any alternative solutions or features you've
 
 **Additional context**
 Add any other context or screenshots about the feature request here.
+
+**Plugin?**
+Might this feature be better suited to being a plugin?  Usually features that can be plugins, should be.

.github/ISSUE_TEMPLATE/plugin-request-template.md

+---
+name: Plugin request template
+about: Suggest a plugin for Etherpad
+title: ''
+labels: Plugin Request
+assignees: JohnMcLear
+
+---
+
+* * *
+
+name: Plugin request
+about: Suggest a plugin for this project
+title: ''
+labels: plugin request
+assignees: 
+
+* * *
+
+**Is your plugin request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when (...)
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the plugin request here.

.github/ISSUE_TEMPLATE/security-issue.md

+---
+name: Security issue
+about: Notify the Etherpad foundation of a Security issue
+title: ''
+labels: security
+assignees: ''
+
+---
+
+Please email contact@etherpad.org with details of the security issue prior to posting here.

.github/ISSUE_TEMPLATE/security.md

+* * *
+
+name: Security notification
+about: Disclose a security issue in Etherpad
+title: ''
+labels: security
+assignees: 
+
+* * *
+
+**Our Security disclosure process**
+1. Please email contact@etherpad.org with detials of the exploit including steps to replicate.
+1. Once confirmed we will provide a confirmation, patch and CVE details.

.github/stale.yml

+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 60
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+  - Bug
+  - Serious Bug
+  - Minor bug
+  - Black hole bug
+  - Special case Bug
+  - Upstream bug
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

.github/workflows/codeql-analysis.yml

+name: "CodeQL"
+
+on:
+  push:
+    branches: [develop, master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [develop]
+  schedule:
+    - cron: '0 13 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      # Override language selection by uncommenting this and choosing your languages
+      # with:
+      #   languages: go, javascript, csharp, python, cpp, java
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.travis.yml

@@ -6,13 +6,17 @@ node_js:
 services:
   - docker
 
+cache: false
+
+before_install:
+  - sudo add-apt-repository -y ppa:libreoffice/ppa
+  - sudo apt-get update
+  - sudo apt-get -y install libreoffice
+  - sudo apt-get -y install libreoffice-pdfimport
+
 install:
   - "bin/installDeps.sh"
   - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
-  - "npm install ep_test_line_attrib"
-
-before_script:
-  - "tests/frontend/travis/sauce_tunnel.sh"
 
 script:
   - "tests/frontend/travis/runner.sh"
@@ -24,25 +28,39 @@ env:
 
 jobs:
   include:
+    # we can only frontend tests from the ether/ organization and not from forks.
+    # To request tests to be run ask a maintainer to fork your repo to ether/
+    - if: fork = false
+      name: "Test the Frontend"
+      install:
+        #FIXME
+        - "sed 's/\"loglevel\": \"INFO\",/\"loglevel\": \"WARN\",/g' settings.json.template > settings.json"
+        - "tests/frontend/travis/sauce_tunnel.sh"
+        - "bin/installDeps.sh"
+        - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
+      script:
+        - "tests/frontend/travis/runner.sh"
     - name: "Run the Backend tests"
       install:
+        - "bin/installDeps.sh"
         - "cd src && npm install && cd -"
       script:
         - "tests/frontend/travis/runnerBackend.sh"
-    - name: "Test the Frontend"
+## Temporarily commented out the Dockerfile tests
+#    - name: "Test the Dockerfile"
+#      install:
+#        - "cd src && npm install && cd -"
+#      script:
+#        - "docker build -t etherpad:test ."
+#        - "docker run -d -p 9001:9001 etherpad:test && sleep 3"
+#        - "cd src && npm run test-container"
+    - name: "Load test Etherpad"
       install:
         - "bin/installDeps.sh"
-        - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
-        - "npm install ep_test_line_attrib"
-      script:
-        - "tests/frontend/travis/runner.sh"
-    - name: "Test the Dockerfile"
-      install:
         - "cd src && npm install && cd -"
+        - "npm install -g etherpad-load-test"
       script:
-        - "docker build -t etherpad:test ."
-        - "docker run -d -p 9001:9001 etherpad:test && sleep 3"
-        - "cd src && npm run test-container"
+        - "tests/frontend/travis/runnerLoadTest.sh"
 
 notifications:
   irc:

CHANGELOG.md

+# Develop -- TODO Change to 1.8.x.
+* ...
+
+# 1.8.5
+* IMPORTANT DROP OF SUPPORT: Drop support for IE.  Browsers now need async/await.
+* IMPORTANT SECURITY: Rate limit Commits when env=production
+* SECURITY: Non completed uploads no longer crash Etherpad
+* SECURITY: Log authentication requests
+* FEATURE: Support ES6 (migrate from Uglify-JS to Terser)
+* FEATURE: Improve support for non-cookie enabled browsers
+* FEATURE: New hooks for ``index.html``
+* FEATURE: New script to delete sessions.
+* FEATURE: New setting to allow import withing an author session on a pad
+* FEATURE: Checks Etherpad version on startup and notifies if update is available.  Also available in ``/admin`` interface.
+* FEATURE: Timeslider updates pad location to most recent edit
+* MINOR: Outdent UL/LI items on removal of list item
+* MINOR: Various UL/LI import/export bugs
+* MINOR: PDF export fix
+* MINOR: Front end tests no longer run (and subsequently error) on pull requests
+* MINOR: Fix issue with </li> closing a list before it opens
+* MINOR: Fix bug where large pads would fire a console error in timeslider
+* MINOR: Fix ?showChat URL param issue
+* MINOR: Issue where timeslider URI fails to be correct if padID is numeric
+* MINOR: Include prompt for clear authorship when entire document is selected
+* MINOR: Include full document aText every 100 revisions to make pad restoration on database curruption achievable
+* MINOR: Several Colibris CSS fixes
+* MINOR: Use mime library for mime types instead of hard-coded.
+* MINOR: Don't show "new pad button" if instance is read only
+* MINOR: Use latest NodeJS when doing Windows build
+* MINOR: Change disconnect logic to reconnect instead of silently failing
+* MINOR: Update SocketIO, async, jQuery and Mocha which were stuck due to stale code.
+* MINOR: Rewrite the majority of the ``bin`` scripts to use more modern syntax
+* MINOR: Improved CSS anomation through prefers-reduced-motion
+* PERFORMANCE: Use workers (where possible) to minify CSS/JS on first page request.  This improves initial startup times.
+* PERFORMANCE: Cache EJS files improving page load speed when maxAge > 0.
+* PERFORMANCE: Fix performance for large pads
+* TESTS: Additional test coverage for OL/LI/Import/Export
+* TESTS: Include Simulated Load Testing in CI.
+* TESTS: Include content collector tests to test contentcollector.js logic external to pad dependents.
+* TESTS: Include fuzzing import test.
+* TESTS: Ensure CI is no longer using any cache
+* TESTS: Fix various tests...
+* TESTS: Various additional Travis testing including libreoffice import/export
+
 # 1.8.4
 * FIX: fix a performance regression on MySQL introduced in 1.8.3
 * FIX: when running behind a reverse proxy and exposed in an inner directory, fonts and toolbar icons should now be visible. This is a regression introduced in 1.8.3

Dockerfile

@@ -42,7 +42,7 @@ RUN bin/installDeps.sh && \
 #
 # Bash trick: in the for loop ${ETHERPAD_PLUGINS} is NOT quoted, in order to be
 # able to split at spaces.
-RUN for PLUGIN_NAME in ${ETHERPAD_PLUGINS}; do npm install "${PLUGIN_NAME}"; done
+RUN for PLUGIN_NAME in ${ETHERPAD_PLUGINS}; do npm install "${PLUGIN_NAME}" || exit 1; done
 
 # Copy the configuration file.
 COPY --chown=etherpad:0 ./settings.json.docker /opt/etherpad-lite/settings.json

README.md

 # A real-time collaborative editor for the web
 <a href="https://hub.docker.com/r/etherpad/etherpad"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/etherpad/etherpad"></a>
-[![Travis (.org)](https://img.shields.io/travis/ether/etherpad-lite)](https://travis-ci.org/github/ether/etherpad-lite)
+
+[![Travis (.org)](https://api.travis-ci.org/ether/etherpad-lite.svg?branch=develop)](https://travis-ci.org/github/ether/etherpad-lite)
+
 ![Demo Etherpad Animated Jif](doc/images/etherpad_demo.gif "Etherpad in action")
 
 # About
@@ -92,6 +94,13 @@ If you prefer, `ep_hash_auth` also gives you the option of storing the users in
 
 Etherpad is very customizable through plugins. Instructions for installing themes and plugins can be found in [the plugin wiki article](https://github.com/ether/etherpad-lite/wiki/Available-Plugins).
 
+## Getting the full features
+Run the following command in your Etherpad folder to get all of the features visible in the demo gif:
+
+```
+npm install ep_headings2 ep_markdown ep_comments_page ep_align ep_page_view ep_font_color ep_webrtc ep_embedded_hyperlinks2
+```
+
 ## Customize the style with skin variants
 
 Open <http://127.0.0.1:9001/p/test#skinvariantsbuilder> in your browser and start playing !
@@ -130,10 +139,12 @@ OpenAPI (previously swagger) definitions for the API are exposed under `/api/ope
 There is a [jQuery plugin](https://github.com/ether/etherpad-lite-jquery-plugin) that helps you to embed Pads into your website.
 
 # Plugin Framework
-Etherpad offers a plugin framework, allowing you to easily add your own features. By default your Etherpad is extremely light-weight and it's up to you to customize your experience. Once you have Etherpad installed you should visit the plugin page and take control.
+Etherpad offers a plugin framework, allowing you to easily add your own features. By default your Etherpad is extremely light-weight and it's up to you to customize your experience. Once you have Etherpad installed you should [visit the plugin page](https://static.etherpad.org/) and take control.
 
 # Translations / Localizations  (i18n / l10n)
-Etherpad comes with translations into all languages thanks to the team at TranslateWiki.
+Etherpad comes with translations into all languages thanks to the team at [TranslateWiki](https://translatewiki.net/).
+
+If you require translations in [plugins](https://static.etherpad.org/) please send pull request to each plugin individually.
 
 # FAQ
 Visit the **[FAQ](https://github.com/ether/etherpad-lite/wiki/FAQ)**.

SECURITY.md

+# Security Policy
+
+## Reporting a Vulnerability
+
+Please email contact@etherpad.org to report security related issues.

bin/buildForWindows.sh

 #!/bin/sh
 
-NODE_VERSION="10.20.1"
-
-#Move to the folder where ep-lite is installed
-cd $(dirname $0)
-
-#Was this script started in the bin folder? if yes move out
-if [ -d "../bin" ]; then
-  cd "../"
-fi
-
-#Is wget installed?
-hash wget > /dev/null 2>&1 || {
-  echo "Please install wget" >&2
-  exit 1
-}
-
-#Is zip installed?
-hash zip > /dev/null 2>&1 || {
-  echo "Please install zip" >&2
-  exit 1
-}
-
-#Is zip installed?
-hash unzip > /dev/null 2>&1 || {
-  echo "Please install unzip" >&2
-  exit 1
-}
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+is_cmd() { command -v "$@" >/dev/null 2>&1; }
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Is wget installed?
+is_cmd wget || fatal "Please install wget"
+
+# Is zip installed?
+is_cmd zip || fatal "Please install zip"
+
+# Is zip installed?
+is_cmd unzip || fatal "Please install unzip"
 
 START_FOLDER=$(pwd);
 TMP_FOLDER=$(mktemp -d)
 
-echo "create a clean environment in $TMP_FOLDER..."
-cp -ar . $TMP_FOLDER
-cd $TMP_FOLDER
+log "create a clean environment in $TMP_FOLDER..."
+cp -ar . "$TMP_FOLDER"
+cd "$TMP_FOLDER"
 rm -rf node_modules
 rm -f etherpad-lite-win.zip
 
@@ -41,33 +31,33 @@ rm -f etherpad-lite-win.zip
 # making the windows package smaller
 export NODE_ENV=production
 
-echo "do a normal unix install first..."
+log "do a normal unix install first..."
 bin/installDeps.sh || exit 1
 
-echo "copy the windows settings template..."
+log "copy the windows settings template..."
 cp settings.json.template settings.json
 
-echo "resolve symbolic links..."
+log "resolve symbolic links..."
 cp -rL node_modules node_modules_resolved
 rm -rf node_modules
 mv node_modules_resolved node_modules
 
-echo "download windows node..."
+log "download windows node..."
 cd bin
-wget "https://nodejs.org/dist/v$NODE_VERSION/win-x86/node.exe" -O ../node.exe
+wget "https://nodejs.org/dist/latest-erbium/win-x86/node.exe" -O ../node.exe
 
-echo "remove git history to reduce folder size"
+log "remove git history to reduce folder size"
 rm -rf .git/objects
 
-echo "remove windows jsdom-nocontextify/test folder"
-rm -rf $TMP_FOLDER/src/node_modules/wd/node_modules/request/node_modules/form-data/node_modules/combined-stream/test
-rm -rf $TMP_FOLDER/src/node_modules/nodemailer/node_modules/mailcomposer/node_modules/mimelib/node_modules/encoding/node_modules/iconv-lite/encodings/tables
+log "remove windows jsdom-nocontextify/test folder"
+rm -rf "$TMP_FOLDER"/src/node_modules/wd/node_modules/request/node_modules/form-data/node_modules/combined-stream/test
+rm -rf "$TMP_FOLDER"/src/node_modules/nodemailer/node_modules/mailcomposer/node_modules/mimelib/node_modules/encoding/node_modules/iconv-lite/encodings/tables
 
-echo "create the zip..."
-cd $TMP_FOLDER
-zip -9 -r $START_FOLDER/etherpad-lite-win.zip ./*
+log "create the zip..."
+cd "$TMP_FOLDER"
+zip -9 -r "$START_FOLDER"/etherpad-lite-win.zip ./*
 
-echo "clean up..."
-rm -rf $TMP_FOLDER
+log "clean up..."
+rm -rf "$TMP_FOLDER"
 
-echo "Finished. You can find the zip in the Etherpad root folder, it's called etherpad-lite-win.zip"
+log "Finished. You can find the zip in the Etherpad root folder, it's called etherpad-lite-win.zip"

bin/convert.js

@@ -349,7 +349,7 @@ function convertPad(padId, callback)
 
           //generate new author values
           var authorID = "a." + randomString(16);
-          var authorColorID = authors[i].colorId || Math.floor(Math.random()*32);
+          var authorColorID = authors[i].colorId || Math.floor(Math.random()*(exports.getColorPalette().length));
           var authorName = authors[i].name || null;
 
           //overwrite the authorID of the attribute pool

bin/createUserSession.js

+/*
+ * A tool for generating a test user session which can be used for debugging configs
+ * that require sessions.
+ */
+const m = (f) => __dirname + '/../' + f;
+
+const fs = require('fs');
+const path = require('path');
+const querystring = require('querystring');
+const request = require(m('src/node_modules/request'));
+const settings = require(m('src/node/utils/Settings'));
+const supertest = require(m('src/node_modules/supertest'));
+
+(async () => {
+  const api = supertest('http://'+settings.ip+':'+settings.port);
+
+  const filePath = path.join(__dirname, '../APIKEY.txt');
+  const apikey = fs.readFileSync(filePath,  {encoding: 'utf-8'});
+
+  let res;
+
+  res = await api.get('/api/');
+  const apiVersion = res.body.currentVersion;
+  if (!apiVersion) throw new Error('No version set in API');
+  const uri = (cmd, args) => `/api/${apiVersion}/${cmd}?${querystring.stringify(args)}`;
+
+  res = await api.post(uri('createGroup', {apikey}));
+  if (res.body.code === 1) throw new Error(`Error creating group: ${res.body}`);
+  const groupID = res.body.data.groupID;
+  console.log('groupID', groupID);
+
+  res = await api.post(uri('createGroupPad', {apikey, groupID}));
+  if (res.body.code === 1) throw new Error(`Error creating group pad: ${res.body}`);
+  console.log('Test Pad ID ====> ', res.body.data.padID);
+
+  res = await api.post(uri('createAuthor', {apikey}));
+  if (res.body.code === 1) throw new Error(`Error creating author: ${res.body}`);
+  const authorID = res.body.data.authorID;
+  console.log('authorID', authorID);
+
+  const validUntil = Math.floor(new Date() / 1000) + 60000;
+  console.log('validUntil', validUntil);
+  res = await api.post(uri('createSession', {apikey, groupID, authorID, validUntil}));
+  if (res.body.code === 1) throw new Error(`Error creating session: ${res.body}`);
+  console.log('Session made: ====> create a cookie named sessionID and set the value to',
+              res.body.data.sessionID);
+})();

bin/debugRun.sh

 #!/bin/sh
 
-#Move to the folder where ep-lite is installed
-cd $(dirname $0)
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
 
-#Was this script started in the bin folder? if yes move out