{"type":"doc","content":[{"type":"layoutSection","content":[{"type":"layoutColumn","attrs":{"width":33.33},"content":[{"type":"table","attrs":{"layout":"default"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"#b3d4ff","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Page Navigation:","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#deebff","rowspan":1},"content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{},"macroMetadata":{"macroId":{"value":"4bc05287-20b8-4d16-a08e-8dcf4c51384d"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}}}}]}]}]}]},{"type":"layoutColumn","attrs":{"width":33.33},"content":[{"type":"paragraph"}]},{"type":"layoutColumn","attrs":{"width":33.33},"content":[{"type":"paragraph"}]}]},{"type":"rule"},{"type":"table","attrs":{"layout":"default"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"#ffebe6","rowspan":1},"content":[{"type":"paragraph","content":[{"type":"emoji","attrs":{"id":"0c076ccb-2db3-41d4-8cb2-d1f129633f89","text":":gauge_low:","shortName":":gauge_low:"}},{"text":" ","type":"text"},{"text":"Confidence Level TBD ","type":"text","marks":[{"type":"textColor","attrs":{"color":"#bf2600"}},{"type":"strong"}]},{"text":" This article has not been reviewed for accuracy, timeliness, or completeness. Check that this information is valid before acting on it.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Integrating a \"Hello World\" Job Type","type":"text"}]},{"type":"paragraph","content":[{"text":"For definitions of terminology used, please refer to our ","type":"text"},{"text":"terminology reference","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Docs#general-hysds-terminology"}}]},{"text":".","type":"text"}]},{"type":"paragraph","content":[{"text":"When installation (see ","type":"text"},{"text":"Installation","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Installation"}}]},{"text":") and setup of your HySDS cluster (see ","type":"text"},{"text":"Cluster Setup","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Cluster-Setup"}}]},{"text":") is complete, you can start integrating job types (PGE or non-PGE) into the system. In this tutorial, we integrate our first job type which will simply echo \"Hello World\".","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"On the ","type":"text"},{"text":"mozart","type":"text","marks":[{"type":"code"}]},{"text":" instance, create a directory for your new job type","type":"text"}]},{"type":"codeBlock","content":[{"text":"cd ~/mozart/ops\nmkdir hello_world\ncd hello_world\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Let's create the script that runs our job type. Open up a new file","type":"text"}]},{"type":"codeBlock","content":[{"text":"vi run_hello_world.sh\n","type":"text"}]},{"type":"paragraph","content":[{"text":"and paste the contents below","type":"text"}]},{"type":"codeBlock","content":[{"text":"#!/bin/bash\necho \"Hello World\"\necho \"Hello World to STDERR\" 1>&2\n","type":"text"}]},{"type":"paragraph","content":[{"text":"and save.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Set the script to be executable","type":"text"}]},{"type":"codeBlock","content":[{"text":"chmod 755 run_hello_world.sh\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Test the script by running it","type":"text"}]},{"type":"codeBlock","content":[{"text":"./run_hello_world.sh \nHello World\nHello World to STDERR\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Now let's create the necessary files for integrating this job into your HySDS cluster. Create the ","type":"text"},{"text":"docker","type":"text","marks":[{"type":"code"}]},{"text":" directory and cd into it","type":"text"}]},{"type":"codeBlock","content":[{"text":"mkdir docker\ncd docker\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create a Dockerfile for your job type by opening up a new file ","type":"text"},{"text":"Dockerfile.","type":"text","marks":[{"type":"code"}]}]},{"type":"codeBlock","content":[{"text":"vi Dockerfile.hello_world\n","type":"text"}]},{"type":"paragraph","content":[{"text":"paste the following into it","type":"text"}]},{"type":"codeBlock","content":[{"text":"FROM hysds/pge-base\n\nARG id\nARG gid\n\n# copy job repo\nCOPY . /home/ops/verdi/ops/hello_world\nRUN set -ex \\\n && sudo chown -R ops:ops /home/ops/verdi/ops/hello_world\n\n# set entrypoint\nUSER ops\nWORKDIR /home/ops\nCMD [\"/bin/bash\", \"--login\"]\n","type":"text"}]},{"type":"paragraph","content":[{"text":"and save.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create a ","type":"text"},{"text":"job-spec","type":"text","marks":[{"type":"code"}]},{"text":" file for your job type by opening up a new file ","type":"text"},{"text":"job-spec.json.","type":"text","marks":[{"type":"code"}]}]},{"type":"codeBlock","content":[{"text":"vi job-spec.json.hello_world\n","type":"text"}]},{"type":"paragraph","content":[{"text":"paste the following into it","type":"text"}]},{"type":"codeBlock","content":[{"text":"{\n \"command\": \"/home/ops/verdi/ops/hello_world/run_hello_world.sh\",\n \"recommended-queues\": [ \"job_worker-small\" ],\n \"params\": []\n}\n","type":"text"}]},{"type":"paragraph","content":[{"text":"and save. For more information on the ","type":"text"},{"text":"job-spec","type":"text","marks":[{"type":"code"}]},{"text":" format, check out ","type":"text"},{"text":"Job and HySDS IO Specifications","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Job-and-HySDS-IO-Specifications"}}]},{"text":".","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create a ","type":"text"},{"text":"hysds-io","type":"text","marks":[{"type":"code"}]},{"text":" file for your job type by opening up a new file ","type":"text"},{"text":"hysds-io.json.","type":"text","marks":[{"type":"code"}]}]},{"type":"codeBlock","content":[{"text":"vi hysds-io.json.hello_world\n","type":"text"}]},{"type":"paragraph","content":[{"text":"paste the following into it","type":"text"}]},{"type":"codeBlock","content":[{"text":"{\n \"label\": \"Hello World\",\n \"params\": []\n}\n","type":"text"}]},{"type":"paragraph","content":[{"text":"and save. For more information on the ","type":"text"},{"text":"hysds-io","type":"text","marks":[{"type":"code"}]},{"text":" format, check out ","type":"text"},{"text":"Job and HySDS IO Specifications","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Job-and-HySDS-IO-Specifications"}}]},{"text":".","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Change directory back up to the root of your job type directory","type":"text"}]},{"type":"codeBlock","content":[{"text":"cd ..\n","type":"text"}]},{"type":"paragraph","content":[{"text":"and the layout of the contents should be as follows","type":"text"}]},{"type":"codeBlock","content":[{"text":"|-- docker\n| |-- Dockerfile.hello_world\n| |-- hysds-io.json.hello_world\n| `-- job-spec.json.hello_world\n`-- run_hello_world.sh\n\n1 directory, 4 files\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Now we create a git repository from this directory","type":"text"}]},{"type":"codeBlock","content":[{"text":"git init\ngit add .\ngit status\n# On branch master\n#\n# Initial commit\n#\n# Changes to be committed:\n# (use \"git rm --cached ...\" to unstage)\n#\n# new file: docker/Dockerfile.hello_world\n# new file: docker/hysds-io.json.hello_world\n# new file: docker/job-spec.json.hello_world\n# new file: run_hello_world.sh\ngit commit -a -m \"initial import\"\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create a new repository on github, copy the repo URL, and push your existing hello_world repository. In this example, I created a new repo under my github user account, ","type":"text"},{"text":"https://github.com/pymonger/hello_world.git","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/pymonger/hello_world.git"}}]}]},{"type":"codeBlock","content":[{"text":"git remote add origin https://github.com//hello_world.git\ngit push -u origin master\nCounting objects: 7, done.\nDelta compression using up to 32 threads.\nCompressing objects: 100% (6/6), done.\nWriting objects: 100% (7/7), 794 bytes | 0 bytes/s, done.\nTotal 7 (delta 0), reused 0 (delta 0)\nTo https://github.com//hello_world.git\n * [new branch] master -> master\nBranch master set up to track remote branch master from origin.\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Ensure that your HySDS cluster is up and running","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds status\n","type":"text"}]},{"type":"paragraph","content":[{"text":"If it is not running, run ","type":"text"},{"text":"sds start all","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Now we need to configure our new job type for continuous integration and deployment. Since we're still developing this job type, we'll configure it using the master branch instead of release tags","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds ci add_job -b master https://github.com//hello_world.git s3 \n","type":"text"}]},{"type":"paragraph","content":[{"text":"If jenkins needs to use an OAuth token to access the repo, specify the ","type":"text"},{"text":"-k","type":"text","marks":[{"type":"code"}]},{"text":" option and ensure that ","type":"text"},{"text":"GIT_OAUTH_TOKEN","type":"text","marks":[{"type":"code"}]},{"text":" was specified in ","type":"text"},{"text":"$HOME/.sds/config","type":"text","marks":[{"type":"code"}]},{"text":":","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds ci add_job -b master -k https://github.com//hello_world.git s3 \n","type":"text"}]},{"type":"paragraph","content":[{"text":"You should have output similar to the following","type":"text"}]},{"type":"codeBlock","content":[{"text":"Jenkins job create for:\n\n https://github.com/pymonger/hello_world.git\n\nPlease ensure that a webhook has been configured at:\n\n https://github.com/pymonger/hello_world/settings/hooks\n\nto push 'Push' and 'Repository' events to:\n\n http://gman-dv-ci.hysds.net:8080/github-webhook/\n\nIf you've configured a Jenkins account with an OAuth credential and full \naccess to your github account, this may be done automatically for you.\n","type":"text"}]},{"type":"paragraph","content":[{"text":"Follow these instructions to configure the webhook on github so that it can trigger rebuild and deployment.","type":"text"}]},{"type":"paragraph","content":[{"text":"Note: If you run into issues like :","type":"text"}]},{"type":"codeBlock","content":[{"text":"[192.168.183.14] run: java -jar /home/ops/jenkins/war/WEB-INF/jenkins-cli.jar -s http://localhost:8080 -http -auth mkarim:9080cc69e123055d68ee19ff0dfa098a reload-configuration\n[192.168.183.14] out: No such command: -http\n[192.168.183.14] out:\n","type":"text"}]},{"type":"paragraph","content":[{"text":"you may have a jenkins-cli.jar version that does not support -http, -auth parameters. You can either get a latest jenkins-cli.jar, or can make the following change line in cluster.py to make it work:","type":"text"}]},{"type":"codeBlock","content":[{"text":"run('java -jar %s/war/WEB-INF/jenkins-cli.jar -s http://localhost:8080 -http -auth %s:%s reload-configuration' %(ctx['JENKINS_DIR'], juser,jkey))\n","type":"text"}]},{"type":"paragraph","content":[{"text":"to:","type":"text"}]},{"type":"codeBlock","content":[{"text":"run('java -jar %s/war/WEB-INF/jenkins-cli.jar -s http://localhost:8080 reload-configuration' % ctx['JENKINS_DIR'])\n","type":"text"}]},{"type":"paragraph","content":[{"text":"If the build fails with following error (aws encryption type) :","type":"text"}]},{"type":"codeBlock","content":[{"text":" out: aws: error: argument --sse: ignored explicit argument 'AES256'\n","type":"text"}]},{"type":"paragraph","content":[{"text":"remove the encryption type value as AES256 is default type in cluster.py:","type":"text"}]},{"type":"codeBlock","content":[{"text":" run('aws s3 cp --sse=AES256 %s s3://%s/' % (tar_file, ctx['CODE_BUCKET']))\n","type":"text"}]},{"type":"paragraph","content":[{"text":"to:","type":"text"}]},{"type":"codeBlock","content":[{"text":" run('aws s3 cp --sse %s s3://%s/' % (tar_file, ctx['CODE_BUCKET']))\n","type":"text"}]},{"type":"paragraph","content":[{"text":"if your build fails with import error for python-pyasn1 or rsa:","type":"text"}]},{"type":"codeBlock","content":[{"text":"ImportError: cannot import name opentype\n[ERROR] Failed to make metadata and store container for: container-hello_world:master\n","type":"text"}]},{"type":"paragraph","content":[{"text":"you may not have latest version of the packages. use pip to install them (specially in ci and factotum instances):","type":"text"}]},{"type":"codeBlock","content":[{"text":"sudo pip install pyasn1_modules\nsudo pip install rsa\n","type":"text"}]},{"type":"paragraph","content":[{"text":"yum may not install the latest version of pyasn1_modules.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Navigate to your ","type":"text"},{"text":"ci","type":"text","marks":[{"type":"code"}]},{"text":" instance (e.g. http://:8080) to validate that the jenkins job was configured","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineExtension","attrs":{"extensionType":"com.atlassian.confluence.migration","extensionKey":"inline-external-image","parameters":{"src":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/hello_world-jenkins.png","width":"","alt":"jenkins","height":""}}},{"text":"You should see your jenkins job named as \"container-builder__\" (e.g. container-builder_gmanipon_hello_world_master).","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Verify that the jenkins job runs to completion by manually scheduling a build. The jenkins job will build the docker image, publish it to your S3 code bucket, and register the job types into you HySDS cluster. Click on the green arrow and view the console output to validate.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Verify that your new job type was published to your HySDS cluster. Open up a browser and point it to your ","type":"text"},{"text":"mozart","type":"text","marks":[{"type":"code"}]},{"text":" instance's ElasticSearch head plugin (e.g. http://:9200/_plugin/head)*","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineExtension","attrs":{"extensionType":"com.atlassian.confluence.migration","extensionKey":"inline-external-image","parameters":{"src":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/hello_world-head.png","width":"","alt":"head","height":""}}},{"text":"Alternatively, you can query for the the \"Hello World\" job-spec using curl","type":"text"}]},{"type":"codeBlock","content":[{"text":"curl -s localhost:9200/job_specs/job_spec/job-hello_world:master | python -m json.tool\n{\n \"_id\": \"job-hello_world:master\",\n \"_index\": \"job_specs\",\n \"_source\": {\n \"command\": \"/home/ops/verdi/ops/hello_world/run_hello_world.sh\",\n \"container\": \"container-hello_world:master\",\n \"id\": \"job-hello_world:master\",\n \"job-version\": \"master\",\n \"params\": [],\n \"recommended-queues\": [\n \"job_worker-small\"\n ],\n \"resource\": \"jobspec\"\n },\n \"_type\": \"job_spec\",\n \"_version\": 1,\n \"found\": true\n}\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Now let's try to run our hello_world job type using the Dataset Faceted Search interface (aka ","type":"text"},{"text":"tosca","type":"text","marks":[{"type":"code"}]},{"text":") running on the ","type":"text"},{"text":"grq","type":"text","marks":[{"type":"code"}]},{"text":" instance. First, let's ingest a dataset so that we have at least 1 dataset in our catalog. On the ","type":"text"},{"text":"mozart","type":"text","marks":[{"type":"code"}]},{"text":" instance run","type":"text"}]},{"type":"codeBlock","content":[{"text":"source ~/mozart/bin/activate\ncd ~/.sds/files\n~/mozart/ops/hysds/scripts/ingest_dataset.py AOI_sacramento_valley ~/mozart/etc/datasets.json\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Navigate to your ","type":"text"},{"text":"grq","type":"text","marks":[{"type":"code"}]},{"text":" instance (e.g. http:///search)* to validate that the AOI dataset was ingested","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"align-start"},"content":[{"type":"media","attrs":{"alt":"ingest","type":"external","url":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/hello_world-ingest.png"}}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Next click on the \"On-Demand\" button. If you don't see the \"On-Demand\" button, then the AOI dataset wasn't ingested correctly. Otherwise, a modal titled \"On-Demand (Process Results)\" should pop up as below","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"align-start"},"content":[{"type":"media","attrs":{"alt":"on-demand","type":"external","url":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/hello_world-on_demand.png"}}]},{"type":"paragraph","content":[{"text":"Enter the value \"test\" into the \"Tag\" field, select \"Hello World [master]\" for the \"Action\" field, and select \"job_worker-small\" for the \"Queue\" field (should already be selected because it is the \"recommended-queue\" based on our job-spec.json.hello_world that we composed earlier). You can leave the \"Priority\" field at \"0\" for now","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"align-start"},"content":[{"type":"media","attrs":{"alt":"submit","type":"external","url":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/hello_world-submit_on_demand.png"}}]},{"type":"paragraph","content":[{"text":"Click on \"Process Now\".","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"You will then get another modal providing you with a link to the HySDS Resource Management interface (aka ","type":"text"},{"text":"figaro","type":"text","marks":[{"type":"code"}]},{"text":") for the job you just submitted. Click on the link to open up ","type":"text"},{"text":"figaro","type":"text","marks":[{"type":"code"}]},{"text":". If the link doesn't open up correctly, there may be a routing issue with the private vs. public IP address in the ","type":"text"},{"text":"tosca","type":"text","marks":[{"type":"code"}]},{"text":" configuration. Alternately, navigate to your ","type":"text"},{"text":"mozart","type":"text","marks":[{"type":"code"}]},{"text":"instance (e.g. http:///figaro)* to view your job. Click on the \"Queued\" button to facet down to the jobs that are in a \"job-queued\" state","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"align-start"},"content":[{"type":"media","attrs":{"alt":"queued","type":"external","url":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/hello_world-job_queued.png"}}]},{"type":"paragraph","content":[{"text":"For advanced users, navigate to the RabbitMQ Admin interface (e.g. http://:15672/#/queues) running on your ","type":"text"},{"text":"mozart","type":"text","marks":[{"type":"code"}]},{"text":" instance to view the queued jobs and tasks","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"align-start"},"content":[{"type":"media","attrs":{"alt":"rabbitmq","type":"external","url":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/hello_world-rabbitmq.png"}}]},{"type":"paragraph","content":[{"text":"It will be stuck in the ","type":"text"},{"text":"job-queued","type":"text","marks":[{"type":"code"}]},{"text":" because there are no workers configured to process jobs from the \"job_worker-small\" queue.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Let's configure your ","type":"text"},{"text":"factotum","type":"text","marks":[{"type":"code"}]},{"text":" instance to start up a worker that will pull jobs from the \"job_worker-small\" queue. What we will do is copy the default supervisord.conf template for factotum to ","type":"text"},{"text":"~/.sds/files","type":"text","marks":[{"type":"code"}]},{"text":", and modify it to include start up of ","type":"text"},{"text":"celery","type":"text","marks":[{"type":"code"}]},{"text":" workers for the ","type":"text"},{"text":"job_worker-small","type":"text","marks":[{"type":"code"}]},{"text":" queue:","type":"text"}]},{"type":"codeBlock","content":[{"text":"cd ~/.sds/files\ncp ~/mozart/ops/hysds/configs/supervisor/supervisord.conf.factotum .\n","type":"text"}]},{"type":"paragraph","content":[{"text":"Edit the ","type":"text"},{"text":"supervisord.conf.factotum","type":"text","marks":[{"type":"code"}]},{"text":" file","type":"text"}]},{"type":"codeBlock","content":[{"text":"vi ~/.sds/files/supervisord.conf.factotum\n","type":"text"}]},{"type":"paragraph","content":[{"text":"append the following lines to the bottom of the file and save","type":"text"}]},{"type":"codeBlock","content":[{"text":"[program:job_worker-small]\ndirectory={{ OPS_HOME }}/verdi/ops/hysds\nenvironment=HYSDS_ROOT_WORK_DIR=\"/data/work\",\n HYSDS_DATASETS_CFG=\"{{ OPS_HOME }}/verdi/etc/datasets.json\"\ncommand=celery worker --app=hysds --concurrency=1 --loglevel=INFO -Q job_worker-small -n %(program_name)s.%(process_num)02d.%%h -O fair --without-mingle --without-gossip --heartbeat-interval=60\nprocess_name=%(program_name)s-%(process_num)02d\npriority=1\nnumprocs=1\nnumprocs_start=0\nredirect_stderr=true\nstdout_logfile=%(here)s/../log/%(program_name)s-%(process_num)02d.log\nstartsecs=10\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Since there are no jobs running in your cluster, we can safely update the factotum instance","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds stop factotum\n","type":"text"}]},{"type":"paragraph","content":[{"text":"Check the status of ","type":"text"},{"text":"supervisord","type":"text","marks":[{"type":"code"}]},{"text":" on factotum","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds status factotum\n","type":"text"}]},{"type":"paragraph","content":[{"text":"We're ready to update if it shows","type":"text"}]},{"type":"codeBlock","content":[{"text":"########################################\nfactotum\n########################################\n[100.64.106.64] Executing task 'status'\nSupervisord is not running on factotum.\n","type":"text"}]},{"type":"paragraph","content":[{"text":"Finally update and start up factotum:","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds update factotum\nsds start factotum\n","type":"text"}]},{"type":"paragraph","content":[{"text":"Rerun ","type":"text"},{"text":"sds status factotum","type":"text","marks":[{"type":"code"}]},{"text":" to verify that your new worker for ","type":"text"},{"text":"job_worker-small","type":"text","marks":[{"type":"code"}]},{"text":" is ","type":"text"},{"text":"RUNNING","type":"text","marks":[{"type":"code"}]}]},{"type":"codeBlock","content":[{"text":"[172.31.6.173] out: job_worker-small:job_worker-small-00 RUNNING pid 20200, uptime 0:00:25\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Navigate to your ","type":"text"},{"text":"mozart","type":"text","marks":[{"type":"code"}]},{"text":" instance (e.g. http:///figaro)* to view your job again. Click on the \"Started\" button to facet down to the jobs that are in a \"job-started\" state","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"align-start"},"content":[{"type":"media","attrs":{"alt":"started","type":"external","url":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/hello_world-job_started.png"}}]},{"type":"paragraph","content":[{"text":"When the job is finished executing, you should have a job that is in a \"job-completed\" state","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"align-start"},"content":[{"type":"media","attrs":{"alt":"completed","type":"external","url":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/hello_world-job_completed.png"}}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Click on the \"work directory\" link to view the live work directory on factotum. All STDOUT output will go into the ","type":"text"},{"text":"_stdout.txt","type":"text","marks":[{"type":"code"}]},{"text":" file and all STDERR output will go into the ","type":"text"},{"text":"_stderr.txt","type":"text","marks":[{"type":"code"}]},{"text":". Here's what ","type":"text"},{"text":"_stdout.txt","type":"text","marks":[{"type":"code"}]},{"text":" should show","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"align-start"},"content":[{"type":"media","attrs":{"alt":"stdout","type":"external","url":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/hello_world-stdout.png"}}]},{"type":"paragraph","content":[{"text":"and what the ","type":"text"},{"text":"_stderr.txt","type":"text","marks":[{"type":"code"}]},{"text":" should show","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"align-start"},"content":[{"type":"media","attrs":{"alt":"stderr","type":"external","url":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/hello_world-stderr.png"}}]},{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"If you run into issues like:","type":"text"}]},{"type":"codeBlock","content":[{"text":"Failed to download http:///products/incoming/v0.1/: 404 Client Error: Not Found for url: …\n","type":"text"}]},{"type":"paragraph","content":[{"text":"it could be due to permission issue in aws. Check the permission tab in 'grfn-v2-ops-product-bucket' to ensure you have the same configuration, specially, the policy file (dont forget to edit the bucket name in your policy file). Also download the index.html file from there and add to your bucket (do necessary edits to reflect your bucket)","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"That's it!","type":"text"}]},{"type":"paragraph","content":[{"text":"Congratulations, you've integrated a new job type into your HySDS cluster and successfully executed it.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Next Steps","type":"text"}]},{"type":"paragraph","content":[{"text":"Now that you've integrated your first job type, continue on to ","type":"text"},{"text":"Hello Dataset","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Hello-Dataset"}}]},{"text":" to integrate a dataset produced by this job type.","type":"text"}]},{"type":"rule"},{"type":"paragraph","content":[{"text":"*With the OPS_USER and OPS_PASSWORD, you can use the https (e.g. https:// ) protocol to gain access to the url.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":" Pages ","type":"text"},{"text":"24","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ffffff"}}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Home","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Home"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Downloads","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/releases"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Docs","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Docs"}}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Installation","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Installation"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Cluster Setup","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Cluster-Setup"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Hello World","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Hello-World"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Hello Dataset","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Hello-Dataset"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Upgrade","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Upgrade"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Datasets Reference","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Datasets"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Puppet Automation","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Puppet-Automation"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Resource Management","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Resource-Management"}}]},{"text":"(Mozart)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Mozart Watchdog Notifications","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Mozart-Watchdog-Notifications"}}]},{"text":" (Mozart)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Dataset Search","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Dataset-Search"}}]},{"text":" (Tosca)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Metrics","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Metrics"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"System Light-Weight Jobs","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/System-Light-Weight-Jobs"}}]},{"text":" (Factotum)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Workers","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Workers"}}]},{"text":" (Verdi)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Job and HySDS IO Specifications","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Job-and-HySDS-IO-Specifications"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Preprocessor and Postprocessor Functions","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Preprocessor-and-Postprocessor-Functions"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Workflows","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Workflows"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"System Port Map","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/System-Port-Map"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"FAQs","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/FAQs"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"How Tos","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/How-Tos"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Releases","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Releases"}}]}]}]}]}]}]},{"type":"heading","attrs":{"level":5},"content":[{"text":"Clone this wiki locally","type":"text"}]},{"type":"paragraph"},{"type":"paragraph"},{"type":"rule"},{"type":"layoutSection","content":[{"type":"layoutColumn","attrs":{"width":25.0},"content":[{"type":"table","attrs":{"layout":"default"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"#abf5d1","rowspan":1},"content":[{"type":"paragraph","content":[{"type":"emoji","attrs":{"id":"1f4d6","text":"📖","shortName":":book:"}},{"text":" ","type":"text"},{"text":"Related Articles:","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#e3fcef","rowspan":1},"content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"contentbylabel","parameters":{"macroParams":{"showLabels":{"value":"false"},"max":{"value":"12"},"showSpace":{"value":"false"},"sort":{"value":"title"},"cql":{"value":"label = \"hysds_setup\""}},"macroMetadata":{"macroId":{"value":"e0cca5c9-3b10-452f-add4-86c9c821a014"},"schemaVersion":{"value":"4"},"title":"Content by Label"}}}}]}]}]}]},{"type":"layoutColumn","attrs":{"width":50.0},"content":[{"type":"table","attrs":{"layout":"default"},"content":[{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#c0b6f2","rowspan":1},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"type":"emoji","attrs":{"id":"atlassian-light_bulb_on","text":":light_bulb_on:","shortName":":light_bulb_on:"}},{"text":" Have Questions? Ask a HySDS Developer:","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#eae6ff","rowspan":1},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"Anyone can join our ","type":"text"},{"text":"public Slack channel","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://hysds.slack.com/"}}]},{"text":" ","type":"text","marks":[{"type":"strong"}]},{"text":"to learn more about HySDS. JPL employees can join ","type":"text"},{"text":"#HySDS-Community","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://jpl.slack.com/archives/C99QZ9THS"}}]},{"text":" ","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"type":"emoji","attrs":{"id":"62f2fc57-741d-435e-a311-29c67cf551fb","text":":slack:","shortName":":slack:"}},{"text":" ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#eae6ff","rowspan":1},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"JPLers can also ask HySDS questions at","type":"text"},{"text":" ","type":"text","marks":[{"type":"strong"}]},{"text":"Stack Overflow Enterprise","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://stackoverflow.jpl.nasa.gov/"}}]}]},{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"type":"emoji","attrs":{"id":"fc6d585e-4859-4319-9d76-b0c6554a66a4","text":":Stack_Overflow:","shortName":":Stack_Overflow:"}},{"text":" ","type":"text"}]}]}]}]},{"type":"paragraph"},{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"type":"inlineExtension","attrs":{"extensionType":"com.atlassian.confluence.macro.core","extensionKey":"livesearch","parameters":{"macroParams":{"placeholder":{"value":"Search HySDS Wiki"}},"macroMetadata":{"macroId":{"value":"3d90ba13-71d8-41da-a917-d015f000ccb5"},"schemaVersion":{"value":"1"},"title":"Livesearch"}}}}]}]},{"type":"layoutColumn","attrs":{"width":25.0},"content":[{"type":"table","attrs":{"layout":"default"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"#b3d4ff","rowspan":1},"content":[{"type":"paragraph","content":[{"type":"emoji","attrs":{"id":"1f680","text":"🚀","shortName":":rocket:"}},{"text":" ","type":"text"},{"text":"Page Information:","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#deebff","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Was this page useful? ","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Yes ","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://hysds-core.atlassian.net/wiki/spaces/HYS/pages/773849112/Page+Feedback+Thumbs+Up"}}]},{"type":"emoji","attrs":{"id":"1f44d","text":"👍","shortName":":thumbsup:"}},{"text":" ","type":"text","marks":[{"type":"strong"}]},{"text":"No","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://hysds-core.atlassian.net/wiki/spaces/HYS/pages/773783581/Page+Feedback+Thumbs+Down"}}]},{"text":" ","type":"text","marks":[{"type":"strong"}]},{"type":"emoji","attrs":{"id":"1f44e","text":"👎","shortName":":thumbsdown:"}},{"text":" ","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#deebff","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Contribution History:","type":"text","marks":[{"type":"strong"}]}]},{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"contributors","parameters":{"macroParams":{"mode":{"value":"list"},"showLastTime":{"value":"true"},"order":{"value":"update"}},"macroMetadata":{"macroId":{"value":"a5475675-daf4-48f6-81cf-8b3a9cb2269d"},"schemaVersion":{"value":"1"},"title":"Contributors"}}}}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#deebff","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Subject Matter Expert:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"type":"mention","attrs":{"id":"5cdb24c7beee910dc4e3c681","text":"@Alexander Torres"}},{"text":" ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#deebff","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Find an Error?","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Is this document outdated or inaccurate? Please contact the assigned Page Maintainer:","type":"text"}]},{"type":"paragraph","content":[{"type":"mention","attrs":{"id":"5b99811ad0b4022bdc51d70f","text":"@Lan Dang"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":" ","type":"text"}]}]}]}]}]}]}],"version":1}

Browser not supported