{"type":"doc","content":[{"type":"paragraph","content":[{"text":"Gerald Manipon edited this page on Aug 30, 2018 · ","type":"text"},{"text":"6 revisions","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Upgrade/_history"}}]}]},{"type":"layoutSection","content":[{"type":"layoutColumn","attrs":{"width":33.33},"content":[{"type":"table","attrs":{"layout":"default"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"#b3d4ff","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Page Navigation:","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#deebff","rowspan":1},"content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{},"macroMetadata":{"macroId":{"value":"8541c0bf-8ab8-4efc-b891-e828ad81f55f"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}}}}]}]}]}]},{"type":"layoutColumn","attrs":{"width":33.33},"content":[{"type":"paragraph"}]},{"type":"layoutColumn","attrs":{"width":33.33},"content":[{"type":"paragraph"}]}]},{"type":"table","attrs":{"layout":"default"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"#ffebe6","rowspan":1},"content":[{"type":"paragraph","content":[{"type":"emoji","attrs":{"id":"0c076ccb-2db3-41d4-8cb2-d1f129633f89","text":":gauge_low:","shortName":":gauge_low:"}},{"text":" ","type":"text"},{"text":"Confidence Level TBD ","type":"text","marks":[{"type":"textColor","attrs":{"color":"#bf2600"}},{"type":"strong"}]},{"text":" This article has not been reviewed for accuracy, timeliness, or completeness. Check that this information is valid before acting on it.","type":"text"}]}]}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"Upgrade","type":"text"}]},{"type":"paragraph","content":[{"text":"For definitions of terminology used, please refer to our ","type":"text"},{"text":"terminology reference","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/wiki/Docs#general-hysds-terminology"}}]},{"text":".","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Latest Releases","type":"text"}]},{"type":"paragraph","content":[{"text":"The latest releases are here: ","type":"text"},{"text":"https://github.com/hysds/hysds-framework/releases","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/hysds/hysds-framework/releases"}}]},{"text":". Each was taken from the latest head state of all repos at time of release.","type":"text"}]},{"type":"paragraph","content":[{"text":"Prerequisite - Graceful Shutdown","type":"text"}]},{"type":"paragraph","content":[{"text":"To preserve the state of queued/running HySDS jobs ","type":"text"},{"text":"mozart","type":"text","marks":[{"type":"code"}]},{"text":", the HySDS cluster should be brought down gracefully as follows:","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Turn off all timer based job submission scripts (e.g. crontab on ","type":"text"},{"text":"factotum","type":"text","marks":[{"type":"code"}]},{"text":")","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Log into into your ","type":"text"},{"text":"factotum","type":"text","marks":[{"type":"code"}]},{"text":" instance","type":"text"}]},{"type":"codeBlock","content":[{"text":"ssh -i ops@\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Back up your crontab and remove it to prevent jobs from being submitted during the upgrade process","type":"text"}]},{"type":"codeBlock","content":[{"text":"mkdir ~/crontabs\ncrontab -l > crontab.$(date -u -Iseconds)\ncrontab -r\n","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Gracefully shutdown the workers","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Log into your ","type":"text"},{"text":"mozart","type":"text","marks":[{"type":"code"}]},{"text":" instance","type":"text"}]},{"type":"codeBlock","content":[{"text":"ssh -i ops@\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Shut down the syncer processes that updates job queue metrics (i.e. AWS CloudWatch) which triggers autoscaling of ","type":"text"},{"text":"verdi","type":"text","marks":[{"type":"code"}]},{"text":" workers:","type":"text"}]},{"type":"codeBlock","content":[{"text":"supervisorctl status | grep ^sync_ | awk '{print $1}' | xargs -i -t supervisorctl stop {}\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Cancel consumption of tasks:","type":"text"}]},{"type":"codeBlock","content":[{"text":"sudo rabbitmqctl list_queues 2>&1 | grep -v '^celery' | tail -n +2 | awk '{print $1}' | xargs -i -t celery -A hysds control cancel_consumer {}\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Log into the RabbitMQ admin interface to ensure that all queues show ","type":"text"},{"text":"0","type":"text","marks":[{"type":"code"}]},{"text":" in the ","type":"text"},{"text":"Unacked","type":"text","marks":[{"type":"code"}]},{"text":"column of the ","type":"text"},{"text":"Queues","type":"text","marks":[{"type":"code"}]},{"text":" tab. To show only the job/tasks queues, enter ","type":"text"},{"text":"^(?!celery)","type":"text","marks":[{"type":"code"}]},{"text":" in the filter text box and check the ","type":"text"},{"text":"Regex","type":"text","marks":[{"type":"code"}]},{"text":" checkbox. If there are jobs/tasks currently running, you can either wait for them to complete or kill them manually to retry them later after the upgrade. Below screenshot shows the ","type":"text"},{"text":"Unacked","type":"text","marks":[{"type":"code"}]},{"text":" column with all zeros signifying that there are no jobs/tasks currently running. ","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"align-start"},"content":[{"type":"media","attrs":{"alt":"rabbitmq","type":"external","url":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/upgrade-rabbitmq.png"}}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Log into ","type":"text"},{"text":"figaro","type":"text","marks":[{"type":"code"}]},{"text":" and ensure that it matches what you see in the RabbitMQ admin interface ","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"align-start"},"content":[{"type":"media","attrs":{"alt":"head","type":"external","url":"https://github.com/hysds/hysds-framework/raw/master/wiki_images/upgrade-figaro.png"}}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Your cluster is now ready for the upgrade","type":"text"}]}]}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Upgrade","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Update HySDS core using hysds-framework and sdscli","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Log into your ","type":"text"},{"text":"mozart","type":"text","marks":[{"type":"code"}]},{"text":" instance","type":"text"}]},{"type":"codeBlock","content":[{"text":"ssh -i ops@\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Stop the cluster","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds stop all -f\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Backup your mozart directory","type":"text"}]},{"type":"codeBlock","content":[{"text":"mv ~/mozart ~/mozart.orig\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"If you have it, remove the old ","type":"text"},{"text":"hysds-framework","type":"text","marks":[{"type":"code"}]},{"text":" clone","type":"text"}]},{"type":"codeBlock","content":[{"text":"rm -rf ~/hysds-framework\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Clone the HySDS framework repository and enter it","type":"text"}]},{"type":"codeBlock","content":[{"text":"cd ~\ngit clone https://github.com/hysds/hysds-framework.git\ncd hysds-framework\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Select the HySDS framework release tag you'd like to install for mozart","type":"text"}]},{"type":"codeBlock","content":[{"text":"./install.sh mozart\nHySDS install directory set to /home/ops/mozart\nNew python executable in /home/ops/mozart/bin/python\nInstalling Setuptools............................................done.\nInstalling Pip...................................................done.\nCreated virtualenv at /home/ops/mozart.\n[2017-08-09 19:25:37,789: INFO/main] Github repo URL: https://xxxxxxxx@github.com/api/v3/repos/hysds/hysds-framework/releases\n[2017-08-09 19:25:37,798: INFO/_new_conn] Starting new HTTPS connection (1): github.com\nNo release specified. Use -r RELEASE | --release=RELEASE to install a specific release. Listing available releases:\nv2.0.0-beta.3\nv2.1.0-beta.0\nv2.1.0-beta.1\nv2.1.0-beta.2\nv2.1.0-beta.3\nv2.1.0-beta.4\nv2.1.0-beta.5\nv2.1.0-beta.6\nv2.1.0-beta.7\nv2.1.0-beta.8\nv2.1.0-rc.0\nv2.1.0-rc.1\nv2.1.0-rc.2\nv2.1.0-rc.3\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Install the latest HySDS release (e.g. v2.1.0-rc.3) for the mozart component","type":"text"}]},{"type":"codeBlock","content":[{"text":"./install.sh mozart -r \n\ne.g.\n\n./install.sh mozart -r v2.1.0-rc.3\n","type":"text"}]},{"type":"paragraph","content":[{"text":"You could also install the development version which pulls the master branch of each HySDS repo:","type":"text"}]},{"type":"codeBlock","content":[{"text":"./install.sh mozart -d\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Restore the non-core repositories from the directory backup under ","type":"text"},{"text":"~/mozart.orig/ops","type":"text","marks":[{"type":"code"}]}]},{"type":"codeBlock","content":[{"text":"cd ~/mozart.orig/ops\nfor i in *; do new=~/mozart/ops/$i; if [ ! -e \"$new\" ]; then cp -rp $i $new; fi done\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Update all HySDS components:","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds update all\n","type":"text"}]},{"type":"paragraph","content":[{"text":"If you receive any errors, they will need to be addressed.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"(Optional) Run any adaptation-specific fabric updates (e.g. update_aria_packages)","type":"text"}]},{"type":"codeBlock","content":[{"text":"fab -f ~/.sds/cluster.py -R factotum,verdi update_aria_packages\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Build and ship out updated code/config bundles","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds ship\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Start up the ","type":"text"},{"text":"grq","type":"text","marks":[{"type":"code"}]},{"text":" component and validate that all services come up fine","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds start grq\nsds status grq\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Start up the ","type":"text"},{"text":"mozart","type":"text","marks":[{"type":"code"}]},{"text":" component and validate that all services come up fine","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds start mozart\nsds status mozart\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Start up the ","type":"text"},{"text":"metrics","type":"text","marks":[{"type":"code"}]},{"text":" component and validate that all services come up fine","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds start metrics\nsds status metrics\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"During installation, the latest versions of the ","type":"text"},{"text":"lightweight-jobs","type":"text","marks":[{"type":"code"}]},{"text":" core HySDS package and the ","type":"text"},{"text":"verdi","type":"text","marks":[{"type":"code"}]},{"text":" docker image was downloaded. If the version has has incremented, we import the ","type":"text"},{"text":"lightweight-jobs","type":"text","marks":[{"type":"code"}]},{"text":" package:","type":"text"}]},{"type":"codeBlock","content":[{"text":"cd ~/mozart/pkgs\nsds pkg import container-hysds_lightweight-jobs.*.sdspkg.tar\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Copy the ","type":"text"},{"text":"verdi","type":"text","marks":[{"type":"code"}]},{"text":" docker image to the code bucket (","type":"text"},{"text":"CODE_BUCKET","type":"text","marks":[{"type":"code"}]},{"text":" as specified during ","type":"text"},{"text":"sds configure","type":"text","marks":[{"type":"code"}]},{"text":"). Ensure ","type":"text"},{"text":"VERDI_PRIMER_IMAGE","type":"text","marks":[{"type":"code"}]},{"text":" url is consistent:","type":"text"}]},{"type":"codeBlock","content":[{"text":"aws s3 cp hysds-verdi-latest.tar.gz s3:///hysds-verdi-latest.tar.gz\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Start up the ","type":"text"},{"text":"factotum","type":"text","marks":[{"type":"code"}]},{"text":" component and validate that all services come up fine","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds start factotum\nsds status factotum\n","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"View status of HySDS components and services:","type":"text"}]},{"type":"codeBlock","content":[{"text":"sds status all\n","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Restore all timer based job submission scripts (e.g. crontab on ","type":"text"},{"text":"factotum","type":"text","marks":[{"type":"code"}]},{"text":")","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Log back into ","type":"text"},{"text":"factotum","type":"text","marks":[{"type":"code"}]},{"text":" and restore the crontab","type":"text"}]},{"type":"codeBlock","content":[{"text":"ssh -i ops@\ncd ~/crontabs\ncrontab crontab.","type":"text"}]}]}]}],"version":1}

Browser not supported