Unified Containerizer

.drone.yml

@@ -1,97 +1,130 @@
-compose:
+services:
+  docker-registry:
+    network_mode: host
+    image: registry
+    environment:
+      - STORAGE_PATH=/var/lib/registry
+      - REGISTRY_LOG_LEVEL=error
+    volumes:
+      - /tmp/registry:/var/lib/registry
   zookeeper:
-    net: host
+    network_mode: host
     image: bobrik/zookeeper
     environment:
       - ZK_CONFIG=tickTime=2000,initLimit=10,syncLimit=5,maxClientCnxns=128,forceSync=no,clientPort=2181
       - ZK_ID=1
   mesos-master:
-    net: host
-    image: mesosphere/mesos-master:0.28.1
+    network_mode: host
+    image: mesosphere/mesos-master:1.0.11.0.1-2.0.93.ubuntu1404
     environment:
+      - MESOS_LOGGING_LEVEL=ERROR
+      - MESOS_IP=127.0.0.1
       - MESOS_ZK=zk://127.0.0.1:2181/mesos
       - MESOS_QUORUM=1
-      - MESOS_CLUSTER=docker-compose
+      - MESOS_CLUSTER=test
       - MESOS_REGISTRY=in_memory
-  mesos-slave1:
-    net: host
-    pid: host
-    image: mesosphere/mesos-slave:0.28.1
+  mesos-slave-0:
+    network_mode: host
+    image: lensa/mesos-slave:1.0.11.0.1-2.0.93.ubuntu1404
     environment:
+      - MESOS_LOGGING_LEVEL=ERROR
+      - MESOS_IP=127.0.0.1
       - MESOS_MASTER=zk://127.0.0.1:2181/mesos
-      - MESOS_CONTAINERIZERS=docker,mesos
+      - MESOS_CONTAINERIZERS=mesos
+      - MESOS_IMAGE_PROVIDERS=docker
+      - MESOS_ISOLATION=filesystem/linux,docker/runtime
+      - MESOS_DOCKER_REGISTRY=http://localhost:5000
       - MESOS_PORT=5051
       - MESOS_RESOURCES=cpus(*):0.5;mem(*):1024;ports(*):[11000-11999]
+      - MESOS_WORK_DIR=/tmp/mesos
+      ## To speed up provisioning images, default is copy
+      # MESOS_IMAGE_PROVISIONER_BACKEND=aufs
+      ## To enable docker containerizer too
+      # MESOS_CONTAINERIZERS=mesos,docker
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup
-      - /usr/local/bin/docker:/usr/bin/docker
-      - /var/run/docker.sock:/var/run/docker.sock
+      ## To cache docker downloaded images
+      # /tmp/mesos/store/docker:/tmp/mesos/store/docker
+      ## To enable docker containerizer
+      # /usr/bin/docker:/usr/bin/docker
+      # /var/run/docker.sock:/var/run/docker.sock
     privileged: true
-  mesos-slave2:
-    net: host
-    pid: host
-    image: mesosphere/mesos-slave:0.28.1
+  mesos-slave-1:
+    network_mode: host
+    image: lensa/mesos-slave:1.0.11.0.1-2.0.93.ubuntu1404
     environment:
+      - MESOS_LOGGING_LEVEL=ERROR
+      - MESOS_IP=127.0.0.1
       - MESOS_MASTER=zk://127.0.0.1:2181/mesos
-      - MESOS_CONTAINERIZERS=docker,mesos
+      - MESOS_CONTAINERIZERS=mesos
+      - MESOS_IMAGE_PROVIDERS=docker
+      - MESOS_ISOLATION=filesystem/linux,docker/runtime
+      - MESOS_DOCKER_REGISTRY=http://localhost:5000
       - MESOS_PORT=5052
       - MESOS_RESOURCES=cpus(*):1.0;mem(*):2048;ports(*):[11000-11999]
+      - MESOS_WORK_DIR=/tmp/mesos
+      ## To speed up provisioning images, default is copy
+      # MESOS_IMAGE_PROVISIONER_BACKEND=aufs
+      ## To enable docker containerizer too
+      # MESOS_CONTAINERIZERS=mesos,docker
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup
-      - /usr/local/bin/docker:/usr/bin/docker
-      - /var/run/docker.sock:/var/run/docker.sock
+      ## To cache docker downloaded images
+      # /tmp/mesos/store/docker:/tmp/mesos/store/docker
+      ## To enable docker containerizer
+      # /usr/bin/docker:/usr/bin/docker
+      # /var/run/docker.sock:/var/run/docker.sock
     privileged: true
 
 
-build:
-  docker:
+pipeline:
+  docker-build:
     image: docker:dind
-    privileged: true
+    network_mode: host
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     commands:
-      - docker build -t lensa/satyr:latest .
+      - docker build -t lensa/satyr .
+      - docker tag lensa/satyr localhost:5000/lensa/satyr
+      - docker push localhost:5000/lensa/satyr
 
   test:
-    image: lensa/satyr:latest
-    net: host
+    image: lensa/satyr
+    network_mode: host
     environment:
+      - LIBPROCESS_IP=127.0.0.1
       - MESOS_MASTER=zk://127.0.0.1:2181/mesos
       - ZOOKEEPER_HOST=127.0.0.1:2181
     commands:
-      - export LIBPROCESS_IP=`hostname -I | cut -f1 -d" "`
       - python setup.py test
 
-
-publish:
-  pypi:
-    repository: https://pypi.python.org/pypi
-    username: $$PYPI_USERNAME
-    password: $$PYPI_PASSWORD
-    distributions:
-      - sdist
-    when:
-      event: [tag]
-
-  docker:
-    username: $$DOCKERHUB_USERNAME
-    password: $$DOCKERHUB_PASSWORD
-    email: $$DOCKERHUB_EMAIL
-    repo: lensa/satyr
-    file: Dockerfile
-    tag:
-      - $$TAG
-      - latest
-    when:
-      event: [tag]
-
-  docker:
-    username: $$DOCKERHUB_USERNAME
-    password: $$DOCKERHUB_PASSWORD
-    email: $$DOCKERHUB_EMAIL
+  docker-push:
+    image: plugins/docker
     repo: lensa/satyr
     file: Dockerfile
     tag:
       - master
     when:
       branch: master
+
+
+  # pypi:
+  #   repository: https://pypi.python.org/pypi
+  #   username: $$PYPI_USERNAME
+  #   password: $$PYPI_PASSWORD
+  #   distributions:
+  #     - sdist
+  #   when:
+  #     event: [tag]
+
+  # docker:
+  #   username: $$DOCKERHUB_USERNAME
+  #   password: $$DOCKERHUB_PASSWORD
+  #   email: $$DOCKERHUB_EMAIL
+  #   repo: lensa/satyr
+  #   file: Dockerfile
+  #   tag:
+  #     - $$TAG
+  #     - latest
+  #   when:
+  #     event: [tag]

.drone.yml.sig

@@ -0,0 +1 @@
+eyJhbGciOiJIUzI1NiJ9.c2VydmljZXM6CiAgZG9ja2VyLXJlZ2lzdHJ5OgogICAgbmV0d29ya19tb2RlOiBob3N0CiAgICBpbWFnZTogcmVnaXN0cnkKICAgIGVudmlyb25tZW50OgogICAgICAtIFNUT1JBR0VfUEFUSD0vdmFyL2xpYi9yZWdpc3RyeQogICAgICAtIFJFR0lTVFJZX0xPR19MRVZFTD1lcnJvcgogICAgdm9sdW1lczoKICAgICAgLSAvdG1wL3JlZ2lzdHJ5Oi92YXIvbGliL3JlZ2lzdHJ5CiAgem9va2VlcGVyOgogICAgbmV0d29ya19tb2RlOiBob3N0CiAgICBpbWFnZTogYm9icmlrL3pvb2tlZXBlcgogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gWktfQ09ORklHPXRpY2tUaW1lPTIwMDAsaW5pdExpbWl0PTEwLHN5bmNMaW1pdD01LG1heENsaWVudENueG5zPTEyOCxmb3JjZVN5bmM9bm8sY2xpZW50UG9ydD0yMTgxCiAgICAgIC0gWktfSUQ9MQogIG1lc29zLW1hc3RlcjoKICAgIG5ldHdvcmtfbW9kZTogaG9zdAogICAgaW1hZ2U6IG1lc29zcGhlcmUvbWVzb3MtbWFzdGVyOjEuMC4xMS4wLjEtMi4wLjkzLnVidW50dTE0MDQKICAgIGVudmlyb25tZW50OgogICAgICAtIE1FU09TX0xPR0dJTkdfTEVWRUw9RVJST1IKICAgICAgLSBNRVNPU19JUD0xMjcuMC4wLjEKICAgICAgLSBNRVNPU19aSz16azovLzEyNy4wLjAuMToyMTgxL21lc29zCiAgICAgIC0gTUVTT1NfUVVPUlVNPTEKICAgICAgLSBNRVNPU19DTFVTVEVSPXRlc3QKICAgICAgLSBNRVNPU19SRUdJU1RSWT1pbl9tZW1vcnkKICBtZXNvcy1zbGF2ZS0wOgogICAgbmV0d29ya19tb2RlOiBob3N0CiAgICBpbWFnZTogbGVuc2EvbWVzb3Mtc2xhdmU6MS4wLjExLjAuMS0yLjAuOTMudWJ1bnR1MTQwNAogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gTUVTT1NfTE9HR0lOR19MRVZFTD1FUlJPUgogICAgICAtIE1FU09TX0lQPTEyNy4wLjAuMQogICAgICAtIE1FU09TX01BU1RFUj16azovLzEyNy4wLjAuMToyMTgxL21lc29zCiAgICAgIC0gTUVTT1NfQ09OVEFJTkVSSVpFUlM9bWVzb3MKICAgICAgLSBNRVNPU19JTUFHRV9QUk9WSURFUlM9ZG9ja2VyCiAgICAgIC0gTUVTT1NfSVNPTEFUSU9OPWZpbGVzeXN0ZW0vbGludXgsZG9ja2VyL3J1bnRpbWUKICAgICAgLSBNRVNPU19ET0NLRVJfUkVHSVNUUlk9aHR0cDovL2xvY2FsaG9zdDo1MDAwCiAgICAgIC0gTUVTT1NfUE9SVD01MDUxCiAgICAgIC0gTUVTT1NfUkVTT1VSQ0VTPWNwdXMoKik6MC41O21lbSgqKToxMDI0O3BvcnRzKCopOlsxMTAwMC0xMTk5OV0KICAgICAgLSBNRVNPU19XT1JLX0RJUj0vdG1wL21lc29zCiAgICAgICMjIFRvIHNwZWVkIHVwIHByb3Zpc2lvbmluZyBpbWFnZXMsIGRlZmF1bHQgaXMgY29weQogICAgICAjIE1FU09TX0lNQUdFX1BST1ZJU0lPTkVSX0JBQ0tFTkQ9YXVmcwogICAgICAjIyBUbyBlbmFibGUgZG9ja2VyIGNvbnRhaW5lcml6ZXIgdG9vCiAgICAgICMgTUVTT1NfQ09OVEFJTkVSSVpFUlM9bWVzb3MsZG9ja2VyCiAgICB2b2x1bWVzOgogICAgICAtIC9zeXMvZnMvY2dyb3VwOi9zeXMvZnMvY2dyb3VwCiAgICAgICMjIFRvIGNhY2hlIGRvY2tlciBkb3dubG9hZGVkIGltYWdlcwogICAgICAjIC90bXAvbWVzb3Mvc3RvcmUvZG9ja2VyOi90bXAvbWVzb3Mvc3RvcmUvZG9ja2VyCiAgICAgICMjIFRvIGVuYWJsZSBkb2NrZXIgY29udGFpbmVyaXplcgogICAgICAjIC91c3IvYmluL2RvY2tlcjovdXNyL2Jpbi9kb2NrZXIKICAgICAgIyAvdmFyL3J1bi9kb2NrZXIuc29jazovdmFyL3J1bi9kb2NrZXIuc29jawogICAgcHJpdmlsZWdlZDogdHJ1ZQogIG1lc29zLXNsYXZlLTE6CiAgICBuZXR3b3JrX21vZGU6IGhvc3QKICAgIGltYWdlOiBsZW5zYS9tZXNvcy1zbGF2ZToxLjAuMTEuMC4xLTIuMC45My51YnVudHUxNDA0CiAgICBlbnZpcm9ubWVudDoKICAgICAgLSBNRVNPU19MT0dHSU5HX0xFVkVMPUVSUk9SCiAgICAgIC0gTUVTT1NfSVA9MTI3LjAuMC4xCiAgICAgIC0gTUVTT1NfTUFTVEVSPXprOi8vMTI3LjAuMC4xOjIxODEvbWVzb3MKICAgICAgLSBNRVNPU19DT05UQUlORVJJWkVSUz1tZXNvcwogICAgICAtIE1FU09TX0lNQUdFX1BST1ZJREVSUz1kb2NrZXIKICAgICAgLSBNRVNPU19JU09MQVRJT049ZmlsZXN5c3RlbS9saW51eCxkb2NrZXIvcnVudGltZQogICAgICAtIE1FU09TX0RPQ0tFUl9SRUdJU1RSWT1odHRwOi8vbG9jYWxob3N0OjUwMDAKICAgICAgLSBNRVNPU19QT1JUPTUwNTIKICAgICAgLSBNRVNPU19SRVNPVVJDRVM9Y3B1cygqKToxLjA7bWVtKCopOjIwNDg7cG9ydHMoKik6WzExMDAwLTExOTk5XQogICAgICAtIE1FU09TX1dPUktfRElSPS90bXAvbWVzb3MKICAgICAgIyMgVG8gc3BlZWQgdXAgcHJvdmlzaW9uaW5nIGltYWdlcywgZGVmYXVsdCBpcyBjb3B5CiAgICAgICMgTUVTT1NfSU1BR0VfUFJPVklTSU9ORVJfQkFDS0VORD1hdWZzCiAgICAgICMjIFRvIGVuYWJsZSBkb2NrZXIgY29udGFpbmVyaXplciB0b28KICAgICAgIyBNRVNPU19DT05UQUlORVJJWkVSUz1tZXNvcyxkb2NrZXIKICAgIHZvbHVtZXM6CiAgICAgIC0gL3N5cy9mcy9jZ3JvdXA6L3N5cy9mcy9jZ3JvdXAKICAgICAgIyMgVG8gY2FjaGUgZG9ja2VyIGRvd25sb2FkZWQgaW1hZ2VzCiAgICAgICMgL3RtcC9tZXNvcy9zdG9yZS9kb2NrZXI6L3RtcC9tZXNvcy9zdG9yZS9kb2NrZXIKICAgICAgIyMgVG8gZW5hYmxlIGRvY2tlciBjb250YWluZXJpemVyCiAgICAgICMgL3Vzci9iaW4vZG9ja2VyOi91c3IvYmluL2RvY2tlcgogICAgICAjIC92YXIvcnVuL2RvY2tlci5zb2NrOi92YXIvcnVuL2RvY2tlci5zb2NrCiAgICBwcml2aWxlZ2VkOiB0cnVlCgoKcGlwZWxpbmU6CiAgZG9ja2VyOgogICAgaW1hZ2U6IGRvY2tlcjpkaW5kCiAgICBuZXR3b3JrX21vZGU6IGhvc3QKICAgIHZvbHVtZXM6CiAgICAgIC0gL3Zhci9ydW4vZG9ja2VyLnNvY2s6L3Zhci9ydW4vZG9ja2VyLnNvY2sKICAgIGNvbW1hbmRzOgogICAgICAtIGRvY2tlciBidWlsZCAtdCBsZW5zYS9zYXR5ciAuCiAgICAgIC0gZG9ja2VyIHRhZyBsZW5zYS9zYXR5ciBsb2NhbGhvc3Q6NTAwMC9zYXR5cgogICAgICAtIGRvY2tlciBwdXNoIGxvY2FsaG9zdDo1MDAwL3NhdHlyCgogIHRlc3Q6CiAgICBpbWFnZToga3N6dWNzL21lc29zLWFscGluZTpweXRob24KICAgIG5ldHdvcmtfbW9kZTogaG9zdAogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gTElCUFJPQ0VTU19JUD0xMjcuMC4wLjEKICAgICAgLSBNRVNPU19NQVNURVI9ems6Ly8xMjcuMC4wLjE6MjE4MS9tZXNvcwogICAgICAtIFpPT0tFRVBFUl9IT1NUPTEyNy4wLjAuMToyMTgxCiAgICBjb21tYW5kczoKICAgICAgLSBweXRob24gc2V0dXAucHkgdGVzdAoKCnB1Ymxpc2g6CiAgcHlwaToKICAgIHJlcG9zaXRvcnk6IGh0dHBzOi8vcHlwaS5weXRob24ub3JnL3B5cGkKICAgIHVzZXJuYW1lOiAkJFBZUElfVVNFUk5BTUUKICAgIHBhc3N3b3JkOiAkJFBZUElfUEFTU1dPUkQKICAgIGRpc3RyaWJ1dGlvbnM6CiAgICAgIC0gc2Rpc3QKICAgIHdoZW46CiAgICAgIGV2ZW50OiBbdGFnXQoKICBkb2NrZXI6CiAgICB1c2VybmFtZTogJCRET0NLRVJIVUJfVVNFUk5BTUUKICAgIHBhc3N3b3JkOiAkJERPQ0tFUkhVQl9QQVNTV09SRAogICAgZW1haWw6ICQkRE9DS0VSSFVCX0VNQUlMCiAgICByZXBvOiBsZW5zYS9zYXR5cgogICAgZmlsZTogRG9ja2VyZmlsZQogICAgdGFnOgogICAgICAtICQkVEFHCiAgICAgIC0gbGF0ZXN0CiAgICB3aGVuOgogICAgICBldmVudDogW3RhZ10KCiAgZG9ja2VyOgogICAgdXNlcm5hbWU6ICQkRE9DS0VSSFVCX1VTRVJOQU1FCiAgICBwYXNzd29yZDogJCRET0NLRVJIVUJfUEFTU1dPUkQKICAgIGVtYWlsOiAkJERPQ0tFUkhVQl9FTUFJTAogICAgcmVwbzogbGVuc2Evc2F0eXIKICAgIGZpbGU6IERvY2tlcmZpbGUKICAgIHRhZzoKICAgICAgLSBtYXN0ZXIKICAgIHdoZW46CiAgICAgIGJyYW5jaDogbWFzdGVyCg.WAtZWrsDsKN6LlA8RLARwNrmc98YoUZ4_pPQWuTGf2M

Dockerfile

@@ -1,5 +1,5 @@
-FROM lensa/mesos-python:0.27.0-py2.7-python-2.7
+FROM kszucs/miniconda-mesos
+
+ADD . /opt/satyr
+RUN pip --no-cache-dir install /opt/satyr \
 
-ADD . /satyr
-WORKDIR /satyr
-RUN pip install .

Dockerfile.mesos

@@ -0,0 +1,21 @@
+FROM debian:8
+
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PATH=/opt/conda/bin:$PATH 
+
+RUN echo "deb http://repos.mesosphere.io/debian/ jessie main" > /etc/apt/sources.list.d/mesosphere.list \
+ && apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF \
+ && apt-get -y --fix-missing update \
+ && apt-get -y install --no-install-recommends curl bzip2 mesos ca-certificates \
+ && curl -o /tmp/miniconda.sh https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh \
+ && /bin/bash /tmp/miniconda.sh -f -b -p /opt/conda \
+ && rm /tmp/miniconda.sh \
+ && conda config --add channels conda-forge \
+ && conda update -y --all \
+ && conda install -y protobuf==2.6.1 \
+ && conda clean -a \
+ && mkdir -p /tmp/mesos && cp -r /usr/lib/python2.7/site-packages/mesos* /tmp/mesos \
+ && apt-get remove --purge -y curl bzip2 mesos $(apt-mark showauto) \
+ && apt-get -y install --no-install-recommends libcurl3 libsvn1 libevent-2.0 libevent-openssl-2.0 libevent-pthreads-2.0 \
+ && mv /tmp/mesos/* /opt/conda/lib/python2.7/site-packages/ \
+ && rm -rf /tmp/mesos \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*jessie*

satyr/__init__.py

@@ -2,14 +2,17 @@
 
 import pkg_resources as _pkg_resources
 
-from .scheduler import QueueScheduler
-from .executor import OneOffExecutor
+from .scheduler import QueueScheduler, SchedulerDriver
+from .executor import ThreadExecutor, ProcessExecutor, ExecutorDriver
 from .messages import PythonTask, PythonTaskStatus  # important to register classes
 
 
 __version__ = _pkg_resources.get_distribution('satyr').version
 
 __all__ = ('QueueScheduler',
-           'OneOffExecutor',
+           'SchedulerDriver',
+           'ExecutorDriver',
+           'ThreadExecutor',
+           'ProcessExecutor',
            'PythonTask',
            'PythonTaskStatus')

satyr/apis/futures.py

@@ -5,8 +5,8 @@
 # TODO: change thrown errors to these
 from concurrent.futures import ALL_COMPLETED, CancelledError, TimeoutError
 
-from ..messages import PythonTask
-from ..scheduler import QueueScheduler, Running
+from ..messages import Cpus, Disk, Mem, PythonExecutor, PythonTask
+from ..scheduler import QueueScheduler, SchedulerDriver
 from ..utils import timeout as seconds
 
 __all__ = ('MesosPoolExecutor',
@@ -52,7 +52,6 @@ def result(self, timeout=None):
                 return self.status.data
             else:
                 try:
-                    print(self.status.data)
                     raise self.status.exception
                 except TypeError:
                     raise ValueError(
@@ -71,17 +70,21 @@ def add_done_callback(self, fn):
         raise NotImplementedError()
 
 
-class MesosPoolExecutor(Running):
+class MesosPoolExecutor(SchedulerDriver):
 
     def __init__(self, max_workers=-1, *args, **kwargs):
         self.max_worker = max_workers  # TODO
         self.scheduler = QueueScheduler()
         super(MesosPoolExecutor, self).__init__(
             self.scheduler, *args, **kwargs)
 
-    def submit(self, fn, args=[], kwargs={}, **kwds):
-        task = PythonTask(fn=fn, args=args, kwargs=kwargs,
-                          name=kwds.pop('name', 'futures'), **kwds)
+    def submit(self, fn, args=[], kwargs={}, name='futures',
+               docker='satyr', force_pull=False, envs={}, uris=[],
+               resources=[Cpus(0.1), Mem(128), Disk(0)], **kwds):
+        executor = PythonExecutor(docker=docker, force_pull=force_pull,
+                                  envs=envs, uris=uris)
+        task = PythonTask(name=name, fn=fn, args=args, kwargs=kwargs,
+                          resources=resources, executor=executor, **kwds)
         self.scheduler.submit(task)
         return Future(task)
 

satyr/apis/multiprocessing.py

@@ -2,9 +2,9 @@
 
 import time
 
-from ..messages import PythonTask
+from ..messages import Cpus, Disk, Mem, PythonExecutor, PythonTask
 from ..queue import Queue
-from ..scheduler import QueueScheduler, Running
+from ..scheduler import QueueScheduler, SchedulerDriver
 from ..utils import timeout
 
 __all__ = ('Pool',
@@ -43,7 +43,7 @@ def successful(self):
         return self.status.has_succeeded()
 
 
-class Pool(Running):
+class Pool(SchedulerDriver):
 
     def __init__(self, processes=-1, *args, **kwargs):
         self.processes = processes
@@ -70,8 +70,12 @@ def apply(self, func, args=[], kwds={}, **kwargs):
         result = self.apply_async(func=func, args=args, kwds=kwds, **kwargs)
         return result.get(timeout=-1)
 
-    def apply_async(self, func, args=[], kwds={}, callback=None, **kwargs):
-        task = PythonTask(name=kwargs.pop('name', 'multiprocessing'),
-                          fn=func, args=args, kwargs=kwds, **kwargs)
+    def apply_async(self, func, args=[], kwds={}, name='multiprocessing',
+                    docker='satyr', force_pull=False, envs={}, uris=[],
+                    resources=[Cpus(0.1), Mem(128), Disk(0)], **kwargs):
+        executor = PythonExecutor(docker=docker, force_pull=force_pull,
+                                  envs=envs, uris=uris)
+        task = PythonTask(name=name, fn=func, args=args, kwargs=kwds,
+                          resources=resources, executor=executor, **kwargs)
         self.scheduler.submit(task)
         return AsyncResult(task)

satyr/apis/tests/test_futures.py

@@ -23,8 +23,8 @@ def test_submit():
         assert isinstance(future1, Future)
         assert isinstance(future2, Future)
 
-        assert future1.result(timeout=10) == 3
-        assert future2.result(timeout=10) == 15
+        assert future1.result(timeout=30) == 3
+        assert future2.result(timeout=30) == 15
 
 
 def test_future_states():
@@ -34,7 +34,7 @@ def add(a, b):
 
     with MesosPoolExecutor(name='futures-pool') as executor:
         future = executor.submit(add, [1, 2])
-        with timeout(15):
+        with timeout(30):
             while future.running():
                 time.sleep(0.1)
             assert future.running() is False
@@ -56,7 +56,7 @@ def raiser():
     with MesosPoolExecutor(name='futures-pool') as executor:
         with pytest.raises(RemoteException) as e:
             future1 = executor.submit(raiser, resources=resources)
-            future1.result(timeout=10)
+            future1.result(timeout=30)
             assert isinstance(e.value, ValueError)
 
 
@@ -66,7 +66,7 @@ def raiser():
 
     with MesosPoolExecutor(name='futures-pool') as executor:
         future = executor.submit(raiser)
-        e = future.exception(timeout=10)
+        e = future.exception(timeout=30)
         assert isinstance(e, RemoteException)
         assert isinstance(e, TypeError)
 
@@ -77,10 +77,10 @@ def fn(a, b):
 
     with MesosPoolExecutor(name='futures-pool') as executor:
         futures = [executor.submit(fn, args=[1, i], resources=resources)
-                   for i in range(10)]
-        values = [f.result(timeout=15) for f in futures]
+                   for i in range(4)]
+        values = [f.result(timeout=30) for f in futures]
 
-    assert values == [i + 1 for i in range(10)]
+    assert values == [i + 1 for i in range(4)]
 
 
 def test_map(resources):