From ff541be7a02458ff8fd86d9d7952029da974c97b Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Sat, 22 Apr 2017 21:26:18 -0500 Subject: [PATCH] Added docker files and updated docs. Fixes #990 (#1017) * WIP #990 initial commit of dockerfiles * WIP #990 Everything but CMD working * Added deepforge config, added torch install * Updated the comment describing the image * WIP Added cuda worker docker image * WIP #990 Added wget and blob configuration * WIP #990 Fixed worker connection dockerfile * WIP #990 Fixed docker image issues * WIP #990 Reuse the node_modules directory * WIP #990 Reuse node modules in docker image * WIP #990 Added entrypoint and cmd files * WIP #990 Updated the standalone image * WIP #990 Removed standalone docker file This should be replaced with something like docker compose instead * WIP #990 Added deployment instructions * WIP #990 Added additional comments about volumes, gpu --- .dockerignore | 1 + Dockerfile | 24 ++++ Dockerfile.worker | 65 +++++++++ docs/deployment/dockerized.rst | 45 ++++++ docs/deployment/installation.rst | 134 ------------------ .../native.rst} | 31 +--- docs/deployment/overview.rst | 26 ++++ docs/index.rst | 9 +- 8 files changed, 171 insertions(+), 164 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 Dockerfile.worker create mode 100644 docs/deployment/dockerized.rst delete mode 100644 docs/deployment/installation.rst rename docs/{getting_started/installation.rst => deployment/native.rst} (72%) create mode 100644 docs/deployment/overview.rst diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..07e6e47 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +/node_modules diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3c6aa8b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +# Dockerfile for running the server itself +FROM node:6.10.1 +MAINTAINER Brian Broll + +RUN echo '{"allow_root": true}' > /root/.bowerrc && mkdir -p /root/.config/configstore/ && \ + echo '{}' > /root/.config/configstore/bower-github.json + +RUN mkdir /deepforge +ADD . /deepforge +WORKDIR /deepforge + +RUN cd $(npm root -g)/npm \ + && npm install fs-extra \ + && sed -i -e s/graceful-fs/fs-extra/ -e s/fs.rename/fs.move/ ./lib/utils/rename.js + +RUN ln -s /deepforge/bin/deepforge /usr/local/bin + +EXPOSE 8888 + +# Set up the data storage +RUN deepforge config blob.dir /data/blob && \ + deepforge config mongo.dir /data/db + +CMD ["deepforge", "start", "--server"] diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 0000000..8ecba14 --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,65 @@ +# This has torch and cuda support +FROM kaixhin/cuda-torch +MAINTAINER Brian Broll + +# install nodejs v6 +RUN groupadd --gid 1000 node \ + && useradd --uid 1000 --gid node --shell /bin/bash --create-home node + +# gpg keys listed at https://github.com/nodejs/node#release-team +RUN set -ex \ + && for key in \ + 9554F04D7259F04124DE6B476D5A82AC7E37093B \ + 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ + FD3A5288F042B6850C66B31F09FE44734EB7990E \ + 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ + DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ + B9AE9905FFD7803F25714661B63B535A4C206CA9 \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + 56730D5401028683275BD23C23EFEFE93C4CFFFE \ + ; do \ + gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$key"; \ + done + +ENV NPM_CONFIG_LOGLEVEL info +ENV NODE_VERSION 6.10.1 + +RUN curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.xz" \ + && curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && grep " node-v$NODE_VERSION-linux-x64.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-x64.tar.xz" -C /usr/local --strip-components=1 \ + && rm "node-v$NODE_VERSION-linux-x64.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs + +# install deepforge +RUN echo '{"allow_root": true}' > /root/.bowerrc && mkdir -p /root/.config/configstore/ && \ + echo '{}' > /root/.config/configstore/bower-github.json + +RUN mkdir /deepforge +ADD . /deepforge +WORKDIR /deepforge + +RUN cd $(npm root -g)/npm \ + && npm install fs-extra \ + && sed -i -e s/graceful-fs/fs-extra/ -e s/fs.rename/fs.move/ ./lib/utils/rename.js + +RUN ln -s /deepforge/bin/deepforge /usr/local/bin + +# configure the worker +RUN deepforge config blob.dir /data/blob && \ + deepforge config mongo.dir /data/db && \ + deepforge config worker.cache.useBlob false && \ + deepforge config worker.cache.dir /deepforge/worker-cache && \ + deepforge config torch.dir /root/torch/ && \ + git config --global user.email "deepforge-worker@deepforge.org" && \ + git config --global user.name "deepforge-worker" + +# Update torch +RUN apt-get update && apt-get install sudo wget && \ + . /root/torch/install/bin/torch-activate && \ + cd /root/torch/ && bash /root/torch/update.sh && \ + deepforge update -t + +ENTRYPOINT ["deepforge", "start", "--worker"] +CMD ["http://172.17.0.1:8888"] diff --git a/docs/deployment/dockerized.rst b/docs/deployment/dockerized.rst new file mode 100644 index 0000000..a8b5742 --- /dev/null +++ b/docs/deployment/dockerized.rst @@ -0,0 +1,45 @@ +Dockerized Installation +----------------------- +Each of the components are also available as docker containers. This page outlines the running of each of the main components as docker containers and connecting them as necessary. + +Database +~~~~~~~~ +First, you can start the mongo container using: + +.. code-block:: bash + + docker run -d -v /abs/path/to/data:/data/db mongo + +where :code:`/abs/path/to/data` is the path to the mongo data location on the host. If running the database in a container, you will need to get the ip address of the given container: + +.. code-block:: bash + + docker inspect | grep IPAddr + +The :code:`` is the value returned from the original :code:`docker run` command. + +When running mongo in a docker container, it is important to mount an external volume (using the :code:`-v` flag) to be used for the actual data (otherwise the data will be lost when the container is stopped). + +Server +~~~~~~ +The DeepForge server can be started with + +.. code-block:: bash + + docker run -d -v /home/irishninja/.deepforge/blob:/data/blob \ + -p 8888:8888 -e MONGO_URI=mongodb://172.17.0.2:27017/deepforge \ + deepforge/server + +where :code:`172.17.0.2` is the ip address of the mongo container and :code:`/home/irishninja/.deepforge/blob` is the path to use for binary DeepForge data on the host. Of course, if the mongo instance is locating at a different location, :code:`MONGO_URI` can be set to this address as well. Also, the first port (:code:`8888`) can be replaced with the desired port to expose on the host. + +Worker +~~~~~~ +As workers may require GPU access, they will need to use the nvidia-docker plugin. Workers can be created using + +.. code-block:: bash + + nvidia-docker run -d deepforge/worker http://172.17.0.1:8888 + +where :code:`http://172.17.0.1:8888` is the location of the DeepForge server to which to connect. + +**Note**: The :code:`deepforge/worker` image is packaged with cuda 7.5. Depending upon your hardware and nvidia version, you may need to build your own docker image or run the worker natively. diff --git a/docs/deployment/installation.rst b/docs/deployment/installation.rst deleted file mode 100644 index 167b2f3..0000000 --- a/docs/deployment/installation.rst +++ /dev/null @@ -1,134 +0,0 @@ -Installation -============ - -DeepForge Component Overview ----------------------------- -DeepForge is composed of four main elements: - -- *Server*: Main component hosting all the project information and is connected to by the clients -- *Database*: MongoDB database containing DeepForge, job queue for the workers, etc -- *Worker*: Slave machine performing the actual machine learning computation -- *Client*: The connected browsers working on DeepForge projects. - -Of course, only the *Server*, *Database* (MongoDB) and *Worker* need to be installed. If you are not going to execute any machine learning pipelines, installing the *Worker* can be skipped. - -Preinstallation ---------------- -Installing dependencies -~~~~~~~~~~~~~~~~~~~~~~~ -The following dependencies are required for each component: - -- *Server* (NodeJS v6.2.1) -- *Database* (MongoDB v3.0.7) -- *Worker*: NodeJS v6.2.1 (used for job management logic) and `Torch `_ (this will be installed automatically by the cli when needed) -- *Client*: We recommend using Google Chrome and are not supporting other browsers (for now). In other words, other browsers can be used at your own risk. - -Installation ------------- -Database -~~~~~~~~ -Download and install MongoDB from the `website `_. If you are planning on running MongoDB locally on the same machine as DeepForge, simply start `mongod` and continue to setting up DeepForge. - -If you are planning on running MongoDB remotely, set the environment variable "MONGO_URI" to the URI of the Mongo instance that DeepForge will be using: - -.. code-block:: bash - - MONGO_URI="mongodb://pathToMyMongo.com:27017/myCollection" deepforge start - -Server -~~~~~~ -The DeepForge server is included with the deepforge cli and can be started simply with - -.. code-block:: bash - - deepforge start --server - -By default, DeepForge will start on `http://localhost:8888`. However, the port can be specified with the `--port` option. For example: - -.. code-block:: bash - - deepforge start --server --port 3000 - -Worker -~~~~~~ -The DeepForge worker can be started with - -.. code-block:: bash - - deepforge start --worker - -The worker will install dependencies the first time it is run (including torch, if it is not already installed). - -To connect to a remote deepforge instance, add the url of the DeepForge server: - -.. code-block:: bash - - deepforge start --worker http://myaddress.com:1234 - -Updating -~~~~~~~~ -DeepForge can be updated with the command line interface rather simply: - -.. code-block:: bash - - deepforge update - -By default, this will update both DeepForge and the local torch installation. To only update DeepForge, add the `--server` flag: - -.. code-block:: bash - - deepforge update --server - -For more update options, check out `deepforge update --help`! - -Manual Installation (Development) ---------------------------------- -Installing DeepForge for development is essentially cloning the repository and then using `npm` (node package manager) to run the various start, test, etc, commands (including starting the individual components). The deepforge cli can still be used but must be referenced from `./bin/deepforge`. That is, `deepforge start` becomes `./bin/deepforge start` (from the project root). - -DeepForge Server -~~~~~~~~~~~~~~~~ -First, clone the repository: - -.. code-block:: bash - - git clone https://github.com/dfst/deepforge.git - -Then install the project dependencies: - -.. code-block:: bash - - npm install - -To run all components locally start with - -.. code-block:: bash - - ./bin/deepforge start - -and navigate to `http://localhost:8888` to start using DeepForge! - -Alternatively, if jobs are going to be executed on an external worker, run `./bin/deepforge start -s` locally and navigate to `http://localhost:8888`. - -DeepForge Worker -~~~~~~~~~~~~~~~~ -If you are using `./bin/deepforge start -s` you will need to set up a DeepForge worker (`./bin/deepforge start` starts a local worker for you!). DeepForge workers are slave machines connected to DeepForge which execute the provided jobs. This allows the jobs to access the GPU, etc, and provides a number of benefits over trying to perform deep learning tasks in the browser. - -Once DeepForge is installed on the worker, start it with - -.. code-block:: bash - - ./bin/deepforge start -w - -Note: If you are running the worker on a different machine, put the address of the DeepForge server as an argument to the command. For example: - -.. code-block:: bash - - ./bin/deepforge start -w http://myaddress.com:1234 - -Updating -~~~~~~~~ -Updating can be done the same as any other git project; that is, by running `git pull` from the project root. Sometimes, the dependencies need to be updated so it is recommended to run `npm install` following `git pull`. - -Configuration -------------- -After installing DeepForge, it can be helpful to check out `configuring DeepForge `_ diff --git a/docs/getting_started/installation.rst b/docs/deployment/native.rst similarity index 72% rename from docs/getting_started/installation.rst rename to docs/deployment/native.rst index 58312b3..ed84f1d 100644 --- a/docs/getting_started/installation.rst +++ b/docs/deployment/native.rst @@ -1,29 +1,6 @@ -Installation -============ +Native Installation +=================== -DeepForge Component Overview ----------------------------- -DeepForge is composed of four main elements: -- *Server*: Main component hosting all the project information and is connected to by the clients -- *Database*: MongoDB database containing DeepForge, job queue for the workers, etc -- *Worker*: Slave machine performing the actual machine learning computation -- *Client*: The connected browsers working on DeepForge projects. - -Of course, only the *Server*, *Database* (MongoDB) and *Worker* need to be installed. If you are not going to execute any machine learning pipelines, installing the *Worker* can be skipped. - -Preinstallation ---------------- -Installing dependencies -~~~~~~~~~~~~~~~~~~~~~~~ -The following dependencies are required for each component: - -- *Server* (NodeJS v6.2.1) -- *Database* (MongoDB v3.0.7) -- *Worker*: NodeJS v6.2.1 (used for job management logic) and `Torch `_ (this will be installed automatically by the cli when needed) -- *Client*: We recommend using Google Chrome and are not supporting other browsers (for now). In other words, other browsers can be used at your own risk. - -Installation ------------- Database ~~~~~~~~ Download and install MongoDB from the `website `_. If you are planning on running MongoDB locally on the same machine as DeepForge, simply start `mongod` and continue to setting up DeepForge. @@ -127,7 +104,3 @@ Note: If you are running the worker on a different machine, put the address of t Updating ~~~~~~~~ Updating can be done the same as any other git project; that is, by running `git pull` from the project root. Sometimes, the dependencies need to be updated so it is recommended to run `npm install` following `git pull`. - -Configuration -------------- -After installing DeepForge, it can be helpful to check out `configuring DeepForge `_ diff --git a/docs/deployment/overview.rst b/docs/deployment/overview.rst new file mode 100644 index 0000000..16cb6d5 --- /dev/null +++ b/docs/deployment/overview.rst @@ -0,0 +1,26 @@ +Overview +======== + +DeepForge Component Overview +---------------------------- +DeepForge is composed of four main elements: + +- *Server*: Main component hosting all the project information and is connected to by the clients +- *Database*: MongoDB database containing DeepForge, job queue for the workers, etc +- *Worker*: Slave machine performing the actual machine learning computation +- *Client*: The connected browsers working on DeepForge projects. + +Of course, only the *Server*, *Database* (MongoDB) and *Worker* need to be installed. If you are not going to execute any machine learning pipelines, installing the *Worker* can be skipped. + +Component Dependencies +---------------------- +The following dependencies are required for each component: + +- *Server* (NodeJS v6.2.1) +- *Database* (MongoDB v3.0.7) +- *Worker*: NodeJS v6.2.1 (used for job management logic) and `Torch `_ (this will be installed automatically by the cli when needed) +- *Client*: We recommend using Google Chrome and are not supporting other browsers (for now). In other words, other browsers can be used at your own risk. + +Configuration +------------- +After installing DeepForge, it can be helpful to check out `configuring DeepForge `_ diff --git a/docs/index.rst b/docs/index.rst index 787b34f..7c16072 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,11 +22,18 @@ Welcome to DeepForge's documentation! fundamentals/custom_layers.rst fundamentals/custom_data_types.rst +.. toctree:: + :maxdepth: 1 + :caption: Deployment + + deployment/overview.rst + deployment/native.rst + deployment/dockerized.rst + .. toctree:: :maxdepth: 1 :caption: Reference - deployment/installation.rst reference/cli.rst reference/configuration.rst reference/operation_feedback.rst