diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..7257b442 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +dist +build +igneous.egg-info +*.so +.eggs +*.pyc +__pycache__ +.cache +test.py +pipeline.py +.pytest_cache diff --git a/.travis.yml b/.travis.yml index ced20e45..22eb2ab5 100755 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,5 @@ language: python python: -- '2.7' -- '3.4' -- '3.5' - '3.6' dist: trusty sudo: required @@ -13,12 +10,12 @@ before_install: - sudo mv secrets /secrets && sudo chown $USER /secrets - sudo mkdir -p $HOME/.cloudvolume - sudo ln -s /secrets $HOME/.cloudvolume/secrets -- $(python version.py) +- $(python igneous_version.py) script: - docker build --tag seunglab/igneous:$APPVERSION . || travis_terminate 1 - docker run -it -v /secrets:/secrets seunglab/igneous:$APPVERSION /bin/sh -c "cd /igneous && py.test -v -x test" || travis_terminate 1; -- docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" || travis_terminate 1; +- echo "$DOCKER_PASSWORD" | docker login -u="$DOCKER_USERNAME" --password-stdin || travis_terminate 1; - if [ "$TRAVIS_BRANCH" == "master" ]; then docker push seunglab/igneous || travis_terminate 1; diff --git a/Dockerfile b/Dockerfile index d44696d3..6d68ee30 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,48 +1,32 @@ -FROM python:3.4 -MAINTAINER William Silversmith -# This image contains private keys, make sure the image is not pushed to docker hub or any public repo. -## INSTALL gsutil -# Prepare the image. -ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y -qq --no-install-recommends \ - apt-utils \ - curl \ - git \ - openssh-client \ - python-openssl \ - python \ - python-pip \ - python-dev \ -# python-h5py \ - python-numpy \ - python-setuptools \ - libboost-all-dev \ -# libhdf5-dev \ - liblzma-dev \ - libgmp-dev \ -# libmpfr-dev \ -# libxml2-dev \ - screen \ - software-properties-common \ - unzip \ - vim \ - wget \ - zlib1g-dev \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -RUN pip install setuptools Cython wheel numpy +FROM python:3.6-slim +LABEL maintainer="William Silversmith, Nico Kemnitz" -# install neuroglancer -RUN mkdir /.ssh ADD ./ /igneous -RUN pip install -r /igneous/requirements.txt -RUN pip install pyasn1 --upgrade -RUN cd /igneous && pip install -e . +RUN apt-get update \ + # Build dependencies + && apt-get install -y -qq --no-install-recommends \ + git \ + libboost-dev \ + build-essential \ + # igneous + runtime dependencies + && cd igneous \ + && pip install --no-cache-dir -r requirements.txt \ + && pip install --no-cache-dir -e . \ + \ + # Cleanup build dependencies + && apt-get remove --purge -y \ + libboost-dev \ + build-essential \ + && apt-get autoremove --purge -y \ + # Cleanup apt + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + # Cleanup temporary python files + && find /usr/local/lib/python3.6 -depth \ + \( \ + \( -type d -a \( -name __pycache__ \) \) \ + -o \ + \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \ + \) -exec rm -rf '{}' + CMD python /igneous/igneous/task_execution.py - - - - - diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..f288702d --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md index aada32c7..440cfd6c 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/seung-lab/igneous.svg?branch=master)](https://travis-ci.org/seung-lab/igneous) +[![Build Status](https://travis-ci.org/seung-lab/igneous.svg?branch=master)](https://travis-ci.org/seung-lab/igneous) [![SfN 2018 Poster](https://img.shields.io/badge/poster-SfN%202018-blue.svg)](https://drive.google.com/open?id=1RKtaAGV2f7F13opnkQfbp6YBqmoD3fZi) # Igneous @@ -19,7 +19,9 @@ git clone git@github.com:seung-lab/igneous.git cd igneous virtualenv venv source venv/bin/activate -pip install -e . +pip install numpy +pip install -r requirements.txt +python setup.py develop ``` The installation will download the dependencies listed in requirements.txt and diff --git a/deployment.yaml b/deployment.yaml index beca4367..4ca3fb0e 100755 --- a/deployment.yaml +++ b/deployment.yaml @@ -23,6 +23,8 @@ spec: resources: requests: memory: 2.5Gi + limits: + memory: 7.0Gi env: - name: QUEUE_TYPE value: aws diff --git a/docker/chunkflow/Dockerfile b/docker/chunkflow/Dockerfile index 1b236c87..00aa8a40 100755 --- a/docker/chunkflow/Dockerfile +++ b/docker/chunkflow/Dockerfile @@ -17,13 +17,13 @@ RUN apt-get update \ build-essential \ # igneous + runtime dependencies && cd igneous \ - && pip install --no-cache-dir -r requirements.txt \ + && pip3 install --no-cache-dir -r requirements.txt \ # we can only use numpy==1.15 due to an issue of skimage: # https://github.com/scikit-image/scikit-image/issues/3551 # the fix was not released yet. # https://github.com/scikit-image/scikit-image/pull/3556 - && pip install numpy==1.15 --no-cache-dir \ - && pip install --no-cache-dir -e . \ + && pip3 install numpy==1.15 --no-cache-dir \ + && pip3 install --no-cache-dir -e . \ \ # Cleanup build dependencies && apt-get remove --purge -y \ @@ -34,7 +34,7 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ # Cleanup temporary python files - && find /usr/local/lib/python3.6 -depth \ + && find /opt/conda/bin/python3.6 -depth \ \( \ \( -type d -a \( -name __pycache__ \) \) \ -o \ diff --git a/igneous/scripts/queue_pickle.py b/igneous/scripts/queue_pickle.py index 1040591f..281e24f6 100755 --- a/igneous/scripts/queue_pickle.py +++ b/igneous/scripts/queue_pickle.py @@ -9,17 +9,29 @@ """ from six.moves import range +import copy from datetime import datetime import json import math import os import click +import numpy as np from tqdm import tqdm +from igneous import tasks +from cloudvolume.lib import touch from taskqueue import TaskQueue +def serializenumpy(obj): + if isinstance(obj, np.ndarray): + return obj.tolist() + return obj + def deserialize(file): + if not os.path.exists(file): + return [] + with open(file, 'r') as f: queue = json.loads(f.read()) return queue @@ -41,24 +53,43 @@ def save(qurl, file): print("Restarting with {} as seed. Tasks: {}".format(file, len(queue))) def save_progress(tq, last_few): + new_queue = [ ] + + for obj in queue: + if type(obj) is dict: + task = obj + else: + task = copy.deepcopy(obj._args) + task['class'] = obj.__class__.__name__ + + new_queue.append(task) + + def serializenumpy(obj): + if isinstance(obj, np.ndarray): + return obj.tolist() + return obj + with open(file, 'w') as f: - f.write(json.dumps(queue)) + f.write(json.dumps(new_queue, default=serializenumpy)) for t in last_few: - tq.delete(t['id']) + tq.delete(t) tq.wait() # Get all task queues - num_lease = 100 + num_lease = 1 last_few = [] with TaskQueue(queue_server='sqs', qurl=qurl) as tq: iters = int(math.ceil(float(tq.enqueued) / float(num_lease))) for i in tqdm(range(iters), desc='Leasing Tasks'): - tasks = tq.raw_lease(numTasks=num_lease, leaseSecs=10) - queue.extend(tasks['items']) - last_few.extend(tasks['items']) + tasks = tq.lease(num_tasks=num_lease, seconds=10) + if type(tasks) is not list: + tasks = [ tasks ] + + queue.extend(tasks) + last_few.extend(tasks) if i % 10 == 0: save_progress(tq, last_few) diff --git a/igneous/secrets.py b/igneous/secrets.py index 5f807720..f3c318e6 100755 --- a/igneous/secrets.py +++ b/igneous/secrets.py @@ -16,7 +16,7 @@ def envval(key, default): QUEUE_NAME = envval('PIPELINE_USER_QUEUE', 'pull-queue') TEST_QUEUE_NAME = envval('TEST_PIPELINE_USER_QUEUE', 'test-pull-queue') -QUEUE_TYPE = envval('QUEUE_TYPE', 'pull-queue') +QUEUE_TYPE = envval('QUEUE_TYPE', 'sqs') SQS_URL = envval('SQS_URL', None) PROJECT_NAME = 'neuromancer-seung-import' APPENGINE_QUEUE_URL = 'https://queue-dot-neuromancer-seung-import.appspot.com' diff --git a/igneous/task_creation.py b/igneous/task_creation.py index d234115c..bfb0b90d 100755 --- a/igneous/task_creation.py +++ b/igneous/task_creation.py @@ -3,19 +3,22 @@ from six.moves import range from itertools import product from functools import reduce +import operator import copy import json import math import os import re +import subprocess import time from time import strftime import numpy as np from tqdm import tqdm +import cloudvolume from cloudvolume import CloudVolume, Storage -from cloudvolume.lib import Vec, Bbox, max2, min2, xyzrange, find_closest_divisor +from cloudvolume.lib import Vec, Bbox, max2, min2, xyzrange, find_closest_divisor, yellow from taskqueue import TaskQueue, MockTaskQueue from igneous import downsample_scales, chunks @@ -28,7 +31,18 @@ ) # from igneous.tasks import BigArrayTask -USER_EMAIL = 'ws9@princeton.edu' # for provenance files +# for provenance files +try: + OPERATOR_CONTACT = subprocess.check_output("git config user.email", shell=True) + OPERATOR_CONTACT = str(OPERATOR_CONTACT.rstrip()) +except: + try: + print(yellow('Unable to determine provenance contact email. Set "git config user.email". Using unix $USER instead.')) + OPERATOR_CONTACT = os.environ['USER'] + except: + print(yellow('$USER was not set. The "owner" field of the provenance file will be blank.')) + OPERATOR_CONTACT = '' + def create_ingest_task(storage, task_queue): """ @@ -100,7 +114,11 @@ def create_info_file_from_build(layer_path, layer_type, resolution, encoding): return vol.info -def create_downsample_scales(layer_path, mip, ds_shape, axis='z', preserve_chunk_size=False): +def create_downsample_scales( + layer_path, mip, ds_shape, axis='z', + preserve_chunk_size=False, chunk_size=None, + encoding=None + ): vol = CloudVolume(layer_path, mip) shape = min2(vol.volume_size, ds_shape) @@ -109,6 +127,9 @@ def create_downsample_scales(layer_path, mip, ds_shape, axis='z', preserve_chunk underlying_mip = (mip + 1) if (mip + 1) in vol.available_mips else mip underlying_shape = vol.mip_underlying(underlying_mip).astype(np.float32) + if chunk_size: + underlying_shape = Vec(*chunk_size).astype(np.float32) + toidx = { 'x': 0, 'y': 1, 'z': 2 } preserved_idx = toidx[axis] underlying_shape[preserved_idx] = float('inf') @@ -121,12 +142,25 @@ def create_downsample_scales(layer_path, mip, ds_shape, axis='z', preserve_chunk scales = scales[1:] # omit (1,1,1) scales = [ list(map(int, vol.downsample_ratio * Vec(*factor3))) for factor3 in scales ] + if len(scales) == 0: + print("WARNING: No scales generated.") + for scale in scales: - vol.add_scale(scale) + vol.add_scale(scale, encoding=encoding, chunk_size=chunk_size) - if preserve_chunk_size: - for i in range(mip + 1, mip + len(scales) + 1): - vol.scales[i]['chunk_sizes'] = vol.scales[mip]['chunk_sizes'] + if chunk_size is None: + if preserve_chunk_size or len(scales) == 0: + chunk_size = vol.scales[mip]['chunk_sizes'] + else: + chunk_size = vol.scales[mip + 1]['chunk_sizes'] + else: + chunk_size = [ chunk_size ] + + if encoding is None: + encoding = vol.scales[mip]['encoding'] + + for i in range(mip + 1, mip + len(scales) + 1): + vol.scales[i]['chunk_sizes'] = chunk_size return vol.commit_info() @@ -134,7 +168,8 @@ def create_downsampling_tasks( task_queue, layer_path, mip=0, fill_missing=False, axis='z', num_mips=5, preserve_chunk_size=True, - sparse=False, bounds=None + sparse=False, bounds=None, chunk_size=None, + encoding=None ): """ mip: Download this mip level, writes to mip levels greater than this one. @@ -145,6 +180,7 @@ def create_downsampling_tasks( preserve_chunk_size: if true, maintain chunk size of starting mip, else, find the closest evenly divisible chunk size to 64,64,64 for this shape and use that. The latter can be useful when mip 0 uses huge chunks and you want to simply visualize the upper mips. + chunk_size: (overrides preserve_chunk_size) force chunk size for new layers to be this. sparse: When downsampling segmentation, if true, don't count black pixels when computing the mode. Useful for e.g. synapses and point labels. bounds: By default, downsample everything, but you can specify restricted bounding boxes @@ -159,7 +195,11 @@ def ds_shape(mip): vol = CloudVolume(layer_path, mip=mip) shape = ds_shape(vol.mip) - vol = create_downsample_scales(layer_path, mip, shape, preserve_chunk_size=preserve_chunk_size) + vol = create_downsample_scales( + layer_path, mip, shape, + preserve_chunk_size=preserve_chunk_size, chunk_size=chunk_size, + encoding=encoding + ) if not preserve_chunk_size: shape = ds_shape(vol.mip + 1) @@ -175,7 +215,8 @@ def ds_shape(mip): print("Volume Bounds: ", vol.bounds) print("Selected ROI: ", bounds) - for startpt in tqdm(xyzrange( bounds.minpt, bounds.maxpt, shape ), desc="Inserting Downsample Tasks"): + total = reduce(operator.mul, np.ceil(bounds.size3() / shape)) + for startpt in tqdm(xyzrange( bounds.minpt, bounds.maxpt, shape ), desc="Inserting Downsample Tasks", total=total): task = DownsampleTask( layer_path=layer_path, mip=vol.mip, @@ -186,6 +227,7 @@ def ds_shape(mip): sparse=sparse, ) task_queue.insert(task) + task_queue.wait('Uploading') vol.provenance.processing.append({ 'method': { @@ -196,34 +238,61 @@ def ds_shape(mip): 'method': 'downsample_with_averaging' if vol.layer_type == 'image' else 'downsample_segmentation', 'sparse': sparse, 'bounds': str(bounds), + 'chunk_size': (list(chunk_size) if chunk_size else None), + 'preserve_chunk_size': preserve_chunk_size, }, - 'by': USER_EMAIL, + 'by': OPERATOR_CONTACT, 'date': strftime('%Y-%m-%d %H:%M %Z'), }) vol.commit_provenance() -def create_deletion_tasks(task_queue, layer_path): +def create_deletion_tasks(task_queue, layer_path, mip=0, num_mips=5): vol = CloudVolume(layer_path) - shape = vol.underlying * 10 + + shape = vol.mip_underlying(mip)[:3] + shape.x *= 2 ** num_mips + shape.y *= 2 ** num_mips - for startpt in tqdm(xyzrange( vol.bounds.minpt, vol.bounds.maxpt, shape ), desc="Inserting Deletion Tasks"): + total_tasks = reduce(operator.mul, np.ceil(vol.bounds.size3() / shape)) + for startpt in tqdm(xyzrange( vol.bounds.minpt, vol.bounds.maxpt, shape ), desc="Inserting Deletion Tasks", total=total_tasks): bounded_shape = min2(shape, vol.bounds.maxpt - startpt) task = DeleteTask( layer_path=layer_path, shape=bounded_shape.clone(), offset=startpt.clone(), + mip=mip, + num_mips=num_mips, ) task_queue.insert(task) task_queue.wait('Uploading DeleteTasks') -def create_meshing_tasks(task_queue, layer_path, mip, shape=Vec(512, 512, 512), - max_simplification_error=40): + vol = CloudVolume(layer_path) + vol.provenance.processing.append({ + 'method': { + 'task': 'DeleteTask', + 'mip': mip, + 'num_mips': num_mips, + 'shape': shape.tolist(), + }, + 'by': OPERATOR_CONTACT, + 'date': strftime('%Y-%m-%d %H:%M %Z'), + }) + vol.commit_provenance() + +def create_meshing_tasks( + task_queue, layer_path, mip, + shape=Vec(512, 512, 64), max_simplification_error=40, + mesh_dir=None, cdn_cache=False + ): shape = Vec(*shape) vol = CloudVolume(layer_path, mip) + if mesh_dir is None: + mesh_dir = 'mesh_mip_{}_err_{}'.format(mip, max_simplification_error) + if not 'mesh' in vol.info: - vol.info['mesh'] = 'mesh_mip_{}_err_{}'.format(mip, max_simplification_error) + vol.info['mesh'] = mesh_dir vol.commit_info() for startpt in tqdm(xyzrange( vol.bounds.minpt, vol.bounds.maxpt, shape ), desc="Inserting Mesh Tasks"): @@ -233,6 +302,8 @@ def create_meshing_tasks(task_queue, layer_path, mip, shape=Vec(512, 512, 512), layer_path, mip=vol.mip, max_simplification_error=max_simplification_error, + mesh_dir=mesh_dir, + cache_control=('' if cdn_cache else 'no-cache'), ) task_queue.insert(task) task_queue.wait('Uploading MeshTasks') @@ -242,9 +313,12 @@ def create_meshing_tasks(task_queue, layer_path, mip, shape=Vec(512, 512, 512), 'task': 'MeshTask', 'layer_path': layer_path, 'mip': vol.mip, - 'shape': shape.tolist(), + 'shape': shape.tolist(), + 'max_simplification_error': max_simplification_error, + 'mesh_dir': mesh_dir, + 'cdn_cache': cdn_cache, }, - 'by': USER_EMAIL, + 'by': OPERATOR_CONTACT, 'date': strftime('%Y-%m-%d %H:%M %Z'), }) vol.commit_provenance() @@ -252,32 +326,45 @@ def create_meshing_tasks(task_queue, layer_path, mip, shape=Vec(512, 512, 512), def create_transfer_tasks( task_queue, src_layer_path, dest_layer_path, chunk_size=None, shape=Vec(2048, 2048, 64), - fill_missing=False, translate=(0,0,0) + fill_missing=False, translate=(0,0,0), + bounds=None, mip=0, preserve_chunk_size=True ): + """ + Transfer data from one data layer to another. It's possible + to transfer from a lower resolution mip level within a given + bounding box. The bounding box should be specified in terms of + the highest resolution. + """ shape = Vec(*shape) - translate = Vec(*translate) - vol = CloudVolume(src_layer_path) + vol = CloudVolume(src_layer_path, mip=mip) + translate = Vec(*translate) // vol.downsample_ratio if not chunk_size: - chunk_size = vol.info['scales'][0]['chunk_sizes'][0] + chunk_size = vol.info['scales'][mip]['chunk_sizes'][0] chunk_size = Vec(*chunk_size) try: - dvol = CloudVolume(dest_layer_path) + dvol = CloudVolume(dest_layer_path, mip=mip) except Exception: # no info file info = copy.deepcopy(vol.info) dvol = CloudVolume(dest_layer_path, info=info) dvol.commit_info() - if chunk_size is not None: - dvol.info['scales'] = dvol.info['scales'][:1] - dvol.info['scales'][0]['chunk_sizes'] = [ chunk_size.tolist() ] - dvol.commit_info() + dvol.info['scales'] = dvol.info['scales'][:mip+1] + dvol.info['scales'][mip]['chunk_sizes'] = [ chunk_size.tolist() ] + dvol.commit_info() - create_downsample_scales(dest_layer_path, mip=0, ds_shape=shape, preserve_chunk_size=True) + create_downsample_scales(dest_layer_path, + mip=mip, ds_shape=shape, preserve_chunk_size=preserve_chunk_size) - bounds = vol.bounds.clone() - for startpt in tqdm(xyzrange( bounds.minpt, bounds.maxpt, shape ), desc="Inserting Transfer Tasks"): + if bounds is None: + bounds = vol.bounds.clone() + else: + bounds = vol.bbox_to_mip(bounds, mip=0, to_mip=mip) + bounds = Bbox.clamp(bounds, vol.bounds) + + total = int(reduce(operator.mul, np.ceil(bounds.size3() / shape))) + for startpt in tqdm(xyzrange( bounds.minpt, bounds.maxpt, shape ), desc="Inserting Transfer Tasks", total=total): task = TransferTask( src_path=src_layer_path, dest_path=dest_layer_path, @@ -285,12 +372,12 @@ def create_transfer_tasks( offset=startpt.clone(), fill_missing=fill_missing, translate=translate, + mip=mip, ) task_queue.insert(task) task_queue.wait('Uploading Transfer Tasks') - dvol = CloudVolume(dest_layer_path) - dvol.provenance.processing.append({ + job_details = { 'method': { 'task': 'TransferTask', 'src': src_layer_path, @@ -298,14 +385,31 @@ def create_transfer_tasks( 'shape': list(map(int, shape)), 'fill_missing': fill_missing, 'translate': list(map(int, translate)), + 'bounds': [ + bounds.minpt.tolist(), + bounds.maxpt.tolist() + ], + 'mip': mip, }, - 'by': USER_EMAIL, + 'by': OPERATOR_CONTACT, 'date': strftime('%Y-%m-%d %H:%M %Z'), - }) + } + + dvol = CloudVolume(dest_layer_path) + dvol.provenance.sources = [ src_layer_path ] + dvol.provenance.processing.append(job_details) dvol.commit_provenance() -def create_contrast_normalization_tasks(task_queue, src_path, dest_path, - shape=None, mip=0, clip_fraction=0.01, fill_missing=False, translate=(0,0,0)): + if vol.path.protocol != 'boss': + vol.provenance.processing.append(job_details) + vol.commit_provenance() + +def create_contrast_normalization_tasks( + task_queue, src_path, dest_path, levels_path=None, + shape=None, mip=0, clip_fraction=0.01, + fill_missing=False, translate=(0,0,0), + minval=None, maxval=None + ): srcvol = CloudVolume(src_path, mip=mip) @@ -332,12 +436,15 @@ def create_contrast_normalization_tasks(task_queue, src_path, dest_path, task = ContrastNormalizationTask( src_path=src_path, dest_path=dest_path, + levels_path=levels_path, shape=task_shape, offset=startpt.clone(), clip_fraction=clip_fraction, mip=mip, fill_missing=fill_missing, translate=translate, + minval=minval, + maxval=maxval, ) task_queue.insert(task) task_queue.wait('Uploading Contrast Normalization Tasks') @@ -351,64 +458,91 @@ def create_contrast_normalization_tasks(task_queue, src_path, dest_path, 'clip_fraction': clip_fraction, 'mip': mip, 'translate': Vec(*translate).tolist(), + 'minval': minval, + 'maxval': maxval, }, - 'by': USER_EMAIL, + 'by': OPERATOR_CONTACT, 'date': strftime('%Y-%m-%d %H:%M %Z'), }) dvol.commit_provenance() -def create_luminance_levels_tasks(task_queue, layer_path, coverage_factor=0.01, shape=None, offset=(0,0,0), mip=0): - vol = CloudVolume(layer_path) +def create_luminance_levels_tasks( + task_queue, layer_path, + levels_path=None, coverage_factor=0.01, + shape=None, offset=(0,0,0), mip=0, bounds=None + ): + """ + Compute per slice luminance level histogram and write them as + $layer_path/levels/$z. Each z file looks like: + + { + "levels": [ 0, 35122, 12, ... ], # 256 indices, index = luminance i.e. 0 is black, 255 is white + "patch_size": [ sx, sy, sz ], # metadata on how large the patches were + "num_patches": 20, # metadata on + "coverage_ratio": 0.011, # actual sampled area on this slice normalized by ROI size. + } + + layer_path: source image to sample from + levels_path: which path to write ./levels/ to (default: $layer_path) + coverage_factor: what fraction of the image to sample + + offset & shape: Allows you to specify an ROI if much of + the edges are black. Defaults to entire image. + mip: int, which mip to work with, default maximum resolution + """ + vol = CloudVolume(layer_path, mip=mip) if shape == None: shape = vol.shape.clone() shape.z = 1 offset = Vec(*offset) + zoffset = offset.clone() - for z in range(vol.bounds.minpt.z, vol.bounds.maxpt.z + 1): - offset.z = z + if bounds is None: + bounds = vol.bounds.clone() + else: + bounds = vol.bbox_to_mip(bounds, mip=0, to_mip=mip) + bounds = Bbox.clamp(bounds, vol.bounds) + + for z in range(bounds.minpt.z, bounds.maxpt.z + 1): + zoffset.z = z task = LuminanceLevelsTask( src_path=layer_path, + levels_path=levels_path, shape=shape, - offset=offset, + offset=zoffset, coverage_factor=coverage_factor, mip=mip, ) task_queue.insert(task) task_queue.wait('Uploading Luminance Levels Tasks') - vol.provenance.processing.append({ - 'method': { - 'task': 'LuminanceLevelsTask', - 'src': layer_path, - 'shape': Vec(*shape).tolist(), - 'offset': Vec(*offset).tolist(), - 'coverage_factor': coverage_factor, - 'mip': mip, - }, - 'by': USER_EMAIL, - 'date': strftime('%Y-%m-%d %H:%M %Z'), - }) - vol.commit_provenance() - -def create_boss_transfer_tasks(task_queue, src_layer_path, dest_layer_path, shape=Vec(1024, 1024, 64)): - # Note: Weird errors with datatype changing to float64 when requesting 2048,2048,64 - # 1024,1024,64 worked nicely though. - shape = Vec(*shape) - vol = CloudVolume(dest_layer_path) - - create_downsample_scales(dest_layer_path, mip=0, ds_shape=shape) + if levels_path: + try: + vol = CloudVolume(levels_path) + except cloudvolume.exceptions.InfoUnavailableError: + vol = CloudVolume(levels_path, info=vol.info) - for startpt in tqdm(xyzrange( vol.bounds.minpt, vol.bounds.maxpt, shape ), desc="Inserting Boss Transfer Tasks"): - task = BossTransferTask( - src_path=src_layer_path, - dest_path=dest_layer_path, - shape=shape.clone(), - offset=startpt.clone(), - ) - task_queue.insert(task) - task_queue.wait('Uploading Boss Transfer Tasks') + if vol.path.protocol != 'boss': + vol.provenance.processing.append({ + 'method': { + 'task': 'LuminanceLevelsTask', + 'src': layer_path, + 'levels_path': levels_path, + 'shape': Vec(*shape).tolist(), + 'offset': Vec(*offset).tolist(), + 'bounds': [ + bounds.minpt.tolist(), + bounds.maxpt.tolist() + ], + 'coverage_factor': coverage_factor, + 'mip': mip, + }, + 'by': OPERATOR_CONTACT, + 'date': strftime('%Y-%m-%d %H:%M %Z'), + }) + vol.commit_provenance() def create_watershed_remap_tasks(task_queue, map_path, src_layer_path, dest_layer_path, shape=Vec(2048, 2048, 64)): shape = Vec(*shape) @@ -435,7 +569,7 @@ def create_watershed_remap_tasks(task_queue, map_path, src_layer_path, dest_laye 'remap_file': map_path, 'shape': list(shape), }, - 'by': USER_EMAIL, + 'by': OPERATOR_CONTACT, 'date': strftime('%Y-%m-%d %H:%M %Z'), }) dvol.commit_provenance() @@ -468,6 +602,7 @@ def create_fixup_downsample_tasks(task_queue, layer_path, points, shape=Vec(2048 ) task_queue.insert(task) task_queue.wait('Uploading') + def create_quantized_affinity_info(src_layer, dest_layer, shape, mip, chunk_size): srcvol = CloudVolume(src_layer) @@ -516,7 +651,7 @@ def create_quantize_tasks( 'fill_missing': fill_missing, 'mip': mip, }, - 'by': USER_EMAIL, + 'by': OPERATOR_CONTACT, 'date': strftime('%Y-%m-%d %H:%M %Z'), }) destvol.commit_provenance() @@ -741,6 +876,7 @@ def create_inference_tasks(task_queue, image_layer_path, convnet_model_path, }) vol.commit_provenance() + def upload_build_chunks(storage, volume, offset=[0, 0, 0], build_chunk_size=[1024,1024,128]): offset = Vec(*offset) shape = Vec(*volume.shape[:3]) diff --git a/igneous/tasks.py b/igneous/tasks.py index d358146e..a5fcf6b4 100755 --- a/igneous/tasks.py +++ b/igneous/tasks.py @@ -211,21 +211,30 @@ def _download_input_chunk(self, bounds): class DeleteTask(RegisteredTask): """Delete a block of images inside a layer on all mip levels.""" - def __init__(self, layer_path, shape, offset): - super(DeleteTask, self).__init__(layer_path, shape, offset) + def __init__(self, layer_path, shape, offset, mip=0, num_mips=5): + super(DeleteTask, self).__init__(layer_path, shape, offset, mip, num_mips) self.layer_path = layer_path self.shape = Vec(*shape) self.offset = Vec(*offset) + self.mip = mip + self.num_mips = num_mips def execute(self): - vol = CloudVolume(self.layer_path) + vol = CloudVolume(self.layer_path, mip=self.mip) highres_bbox = Bbox( self.offset, self.offset + self.shape ) - for mip in vol.available_mips: + + top_mip = min(vol.available_mips[-1], self.mip + self.num_mips) + + for mip in range(self.mip, top_mip + 1): vol.mip = mip - bbox = vol.bbox_to_mip(highres_bbox, 0, mip) + bbox = vol.bbox_to_mip(highres_bbox, self.mip, mip) bbox = bbox.round_to_chunk_size(vol.underlying, offset=vol.bounds.minpt) bbox = Bbox.clamp(bbox, vol.bounds) + + if bbox.volume() == 0: + continue + vol.delete(bbox) @@ -294,15 +303,19 @@ def __init__(self, shape, offset, layer_path, **kwargs): 'mip': kwargs.get('mip', 0), 'simplification_factor': kwargs.get('simplification_factor', 100), 'max_simplification_error': kwargs.get('max_simplification_error', 40), + 'mesh_dir': kwargs.get('mesh_dir', None), 'remap_table': kwargs.get('remap_table', None), 'generate_manifests': kwargs.get('generate_manifests', False), 'low_padding': kwargs.get('low_padding', 0), - 'high_padding': kwargs.get('high_padding', 1) + 'high_padding': kwargs.get('high_padding', 1), + 'parallel_download': kwargs.get('parallel_download', 1), + 'cache_control': kwargs.get('cache_control', None) } def execute(self): self._volume = CloudVolume( - self.layer_path, self.options['mip'], bounded=False) + self.layer_path, self.options['mip'], bounded=False, + parallel=self.options['parallel_download']) self._bounds = Bbox(self.offset, self.shape + self.offset) self._bounds = Bbox.clamp(self._bounds, self._volume.bounds) @@ -316,8 +329,8 @@ def execute(self): data_bounds.maxpt += self.options['high_padding'] self._mesh_dir = None - if 'meshing' in self._volume.info: - self._mesh_dir = self._volume.info['meshing'] + if self.options['mesh_dir'] is not None: + self._mesh_dir = self.options['mesh_dir'] elif 'mesh' in self._volume.info: self._mesh_dir = self._volume.info['mesh'] @@ -358,6 +371,7 @@ def _compute_meshes(self): ), content=self._create_mesh(obj_id), compress=True, + cache_control=self.options['cache_control'] ) if self.options['generate_manifests']: @@ -369,7 +383,8 @@ def _compute_meshes(self): file_path='{}/{}:{}'.format( self._mesh_dir, remapped_id, self.options['lod']), content=json.dumps({"fragments": fragments}), - content_type='application/json' + content_type='application/json', + cache_control=self.options['cache_control'] ) def _create_mesh(self, obj_id): @@ -413,20 +428,18 @@ class MeshManifestTask(RegisteredTask): a single mesh ['0:','1:',..'9:'] """ - def __init__(self, layer_path, prefix, lod=0): + def __init__(self, layer_path, prefix, lod=0, mesh_dir=None): super(MeshManifestTask, self).__init__(layer_path, prefix) self.layer_path = layer_path self.lod = lod self.prefix = prefix + self.mesh_dir = mesh_dir def execute(self): with Storage(self.layer_path) as storage: self._info = json.loads(storage.get_file('info').decode('utf8')) - self.mesh_dir = None - if 'meshing' in self._info: - self.mesh_dir = self._info['meshing'] - elif 'mesh' in self._info: + if self.mesh_dir is None and 'mesh' in self._info: self.mesh_dir = self._info['mesh'] self._generate_manifests(storage) @@ -752,9 +765,17 @@ class ContrastNormalizationTask(RegisteredTask): """TransferTask + Contrast Correction based on LuminanceLevelsTask output.""" # translate = change of origin - def __init__(self, src_path, dest_path, shape, offset, mip, clip_fraction, fill_missing, translate): - super(ContrastNormalizationTask, self).__init__(src_path, dest_path, - shape, offset, mip, clip_fraction, fill_missing, translate) + def __init__( + self, src_path, dest_path, levels_path, shape, + offset, mip, clip_fraction, fill_missing, + translate, minval, maxval + ): + + super(ContrastNormalizationTask, self).__init__( + src_path, dest_path, levels_path, shape, offset, + mip, clip_fraction, fill_missing, translate, + minval, maxval + ) self.src_path = src_path self.dest_path = dest_path self.shape = Vec(*shape) @@ -763,6 +784,10 @@ def __init__(self, src_path, dest_path, shape, offset, mip, clip_fraction, fill_ self.translate = Vec(*translate) self.mip = int(mip) self.clip_fraction = float(clip_fraction) + self.minval = minval + self.maxval = maxval + + self.levels_path = levels_path if levels_path else self.src_path assert 0 <= self.clip_fraction <= 1 @@ -793,7 +818,11 @@ def execute(self): image[:, :, imagez] = img image = np.round(image) - image = np.clip(image, 0.0, maxval).astype(destcv.dtype) + + minval = self.minval if self.minval is not None else 0.0 + maxval = self.maxval if self.maxval is not None else maxval + + image = np.clip(image, minval, maxval).astype(destcv.dtype) bounds += self.translate downsample_and_upload(image, bounds, destcv, self.shape) @@ -831,37 +860,48 @@ def find_section_clamping_values(self, zlevel, lowerfract, upperfract): def fetch_z_levels(self): bounds = Bbox(self.offset, self.shape[:3] + self.offset) - levelfilenames = ['levels/{}/{}'.format(self.mip, z) - for z in range(bounds.minpt.z, bounds.maxpt.z + 1)] - with Storage(self.src_path) as stor: + + levelfilenames = [ + 'levels/{}/{}'.format(self.mip, z) \ + for z in range(bounds.minpt.z, bounds.maxpt.z + 1) + ] + + with Storage(self.levels_path) as stor: levels = stor.get_files(levelfilenames) - errors = [level['filename'] - for level in levels if level['content'] == None] + errors = [ + level['filename'] \ + for level in levels if level['content'] == None + ] + if len(errors): raise Exception(", ".join( errors) + " were not defined. Did you run a LuminanceLevelsTask for these slices?") levels = [( - int(os.path.basename(item['filename'])), - json.loads(item['content'].decode('utf-8')) - ) for item in levels] + int(os.path.basename(item['filename'])), + json.loads(item['content'].decode('utf-8')) + ) for item in levels ] + levels.sort(key=lambda x: x[0]) levels = [x[1] for x in levels] - return [np.array(x['levels'], dtype=np.uint64) for x in levels] + return [ np.array(x['levels'], dtype=np.uint64) for x in levels ] class LuminanceLevelsTask(RegisteredTask): """Generate a frequency count of luminance values by random sampling. Output to $PATH/levels/$MIP/$Z""" - def __init__(self, src_path, shape, offset, coverage_factor, mip): + def __init__(self, src_path, levels_path, shape, offset, coverage_factor, mip): super(LuminanceLevelsTask, self).__init__( - src_path, shape, offset, coverage_factor, mip) + src_path, levels_path, shape, + offset, coverage_factor, mip + ) self.src_path = src_path self.shape = Vec(*shape) self.offset = Vec(*offset) self.coverage_factor = coverage_factor self.mip = int(mip) + self.levels_path = levels_path assert 0 < coverage_factor <= 1, "Coverage Factor must be between 0 and 1" @@ -888,18 +928,19 @@ def execute(self): biggest = bboxes[-1][1] output = { - "levels": levels.tolist(), - "patch_size": biggest.tolist(), - "num_patches": len(bboxes), - "coverage_ratio": covered_area / self.shape.rectVolume(), + "levels": levels.tolist(), + "patch_size": biggest.tolist(), + "num_patches": len(bboxes), + "coverage_ratio": covered_area / self.shape.rectVolume(), } - levels_path = os.path.join(self.src_path, 'levels') - with Storage(levels_path, n_threads=0) as stor: + path = self.levels_path if self.levels_path else self.src_path + path = os.path.join(path, 'levels') + with Storage(path, n_threads=0) as stor: stor.put_json( - file_path="{}/{}".format(self.mip, self.offset.z), - content=output, - cache_control='no-cache' + file_path="{}/{}".format(self.mip, self.offset.z), + content=output, + cache_control='no-cache' ) def select_bounding_boxes(self, dataset_bounds): @@ -935,25 +976,33 @@ def select_bounding_boxes(self, dataset_bounds): class TransferTask(RegisteredTask): # translate = change of origin - def __init__(self, src_path, dest_path, shape, offset, fill_missing, translate): + def __init__( + self, src_path, dest_path, + shape, offset, fill_missing, + translate, mip=0 + ): super(TransferTask, self).__init__( - src_path, dest_path, shape, offset, fill_missing, translate) + src_path, dest_path, shape, + offset, fill_missing, translate, + mip + ) self.src_path = src_path self.dest_path = dest_path self.shape = Vec(*shape) self.offset = Vec(*offset) self.fill_missing = fill_missing self.translate = Vec(*translate) + self.mip = int(mip) def execute(self): - srccv = CloudVolume(self.src_path, fill_missing=self.fill_missing) - destcv = CloudVolume(self.dest_path, fill_missing=self.fill_missing) + srccv = CloudVolume(self.src_path, fill_missing=self.fill_missing, mip=self.mip) + destcv = CloudVolume(self.dest_path, fill_missing=self.fill_missing, mip=self.mip) bounds = Bbox(self.offset, self.shape + self.offset) bounds = Bbox.clamp(bounds, srccv.bounds) image = srccv[bounds.to_slices()] bounds += self.translate - downsample_and_upload(image, bounds, destcv, self.shape) + downsample_and_upload(image, bounds, destcv, self.shape, mip=self.mip) class WatershedRemapTask(RegisteredTask): @@ -1491,6 +1540,3 @@ def _upload_log(self): content=json.dumps(self.log), content_type='application/json' ) - - - diff --git a/version.py b/igneous_version.py similarity index 100% rename from version.py rename to igneous_version.py diff --git a/requirements.txt b/requirements.txt index fa80ab6f..8f3dae7a 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,10 @@ -backports.lzma==0.0.8 -click==6.7 -cryptography==2.3 -#cloud-volume>=0.31.1 -intern +click>=6.7 +cloud-volume>=0.38.0 google-cloud-logging -google-api-python-client==1.6.2 -networkx==2.1 numpy>=1.14.2 -pyasn1 -pyopenssl==17.2.0 +Pillow>=4.2.1 pytest>=3.3.1 -scipy>=1.1.0 task-queue>=0.4.1 -tenacity>=4.10.0 -e git+https://github.com/seung-lab/tqdm.git#egg=tqdm oauth2client==3.0.0 -protobuf>=3.5.1 scikit-image>=0.14.1 diff --git a/setup.py b/setup.py index 2e32980a..5a6f5404 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,11 @@ third_party_dir = './ext/third_party' setuptools.setup( - setup_requires=['pbr'], + setup_requires=['pbr', 'numpy'], + extras_require={ + ':python_version == "2.7"': ['futures'], + ':python_version == "2.6"': ['futures'], + }, pbr=True, ext_modules=[ setuptools.Extension( diff --git a/test/test_tasks.py b/test/test_tasks.py index fc17091d..eca0bb13 100755 --- a/test/test_tasks.py +++ b/test/test_tasks.py @@ -1,4 +1,5 @@ from builtins import range +from collections import defaultdict import json import os.path import shutil @@ -14,6 +15,7 @@ DeleteTask, InferenceTask ) from igneous import downsample, validate_by_template_matching +import igneous.task_creation as tc from igneous.task_creation import create_downsample_scales, create_downsampling_tasks, create_quantized_affinity_info from .layer_harness import delete_layer, create_layer @@ -384,7 +386,40 @@ def test_mesh_manifests(): if os.path.exists(directory): shutil.rmtree(directory) - + +def test_luminance_levels_task(): + directory = '/tmp/removeme/luminance_levels/' + layer_path = 'file://' + directory + + delete_layer(layer_path) + + storage, imgd = create_layer( + size=(256,256,128,1), offset=(0,0,0), + layer_type="image", layer_name='luminance_levels' + ) + + with MockTaskQueue() as tq: + tc.create_luminance_levels_tasks(tq, + layer_path=layer_path, + coverage_factor=0.01, + shape=None, + offset=(0,0,0), + mip=0 + ) + + + gt = [ 0 ] * 256 + for x,y,z in lib.xyzrange( (0,0,0), list(imgd.shape[:2]) + [1] ): + gt[ imgd[x,y,0,0] ] += 1 + + + with open('/tmp/removeme/luminance_levels/levels/0/0', 'rt') as f: + levels = f.read() + + levels = json.loads(levels) + assert levels['coverage_ratio'] == 1.0 + assert levels['levels'] == gt + # def test_watershed(): # return # needs expensive julia stuff enabled in dockerfile