Merge pull request #14 from markusressel/feature/#4_docker

Feature/#4 docker

Author: markusressel

.travis.yml

@@ -16,7 +16,8 @@ python:
 #  - "nightly" # currently points to 3.7-dev
 # command to install dependencies
 install:
-  - pip install .
+  - pip install --upgrade pip
+  - pip install --no-cache-dir .
   #- pip install -r requirements.txt
 # command to run tests
 before_script:

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.6-slim-buster
+
+RUN apt-get update \
+&& apt-get -y install python-skimage git
+
+WORKDIR /app
+
+RUN pip install --no-cache-dir numpy
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+COPY . .
+
+RUN pip install --no-cache-dir .
+
+CMD [ "py-image-dedup", "daemon" ]

docker-compose.yml

@@ -0,0 +1,44 @@
+version: '3.7'
+
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    environment:
+      - discovery.type=single-node
+    networks:
+      - docker-elk
+    restart: on-failure
+  py-image-dedup:
+    build: .
+    environment:
+      # change configuration to your liking
+      - PY_IMAGE_DEDUP_DRY_RUN=False
+      - PY_IMAGE_DEDUP_ANALYSIS_SOURCE_DIRECTORIES=/mnt/source/
+      - PY_IMAGE_DEDUP_ANALYSIS_RECURSIVE=True
+      - PY_IMAGE_DEDUP_ANALYSIS_ACROSS_DIRS=True
+      - PY_IMAGE_DEDUP_ANALYSIS_FILE_EXTENSIONS=.png,.jpg,.jpeg
+      - PY_IMAGE_DEDUP_ANALYSIS_THREADS=8
+      - PY_IMAGE_DEDUP_ANALYSIS_USE_EXIF_DATA=True
+      - PY_IMAGE_DEDUP_DEDUPLICATION_DUPLICATES_TARGET_DIRECTORY=/mnt/duplicates/
+      - PY_IMAGE_DEDUP_ELASTICSEARCH_AUTO_CREATE_INDEX=True
+      - PY_IMAGE_DEDUP_ELASTICSEARCH_HOST=elasticsearch
+      - PY_IMAGE_DEDUP_ELASTICSEARCH_MAX_DISTANCE=0.1
+      - PY_IMAGE_DEDUP_REMOVE_EMPTY_FOLDERS=False
+    volumes:
+      # change this to your local source directory
+      - /mnt/sdb2/Sample:/mnt/source
+      # change this to your local duplicates directory
+      - /mnt/sdb2/py-image-dedup_duplicates:/mnt/duplicates
+    links:
+      - elasticsearch
+    networks:
+      - docker-elk
+    depends_on:
+      - elasticsearch
+    restart: on-failure
+networks:
+  docker-elk:
+    driver: bridge

py_image_dedup/cli.py

@@ -24,6 +24,7 @@
 
 
 @click.group(context_settings=CONTEXT_SETTINGS)
+@click.version_option()
 def cli():
     pass
 

py_image_dedup/config.py

@@ -1,13 +1,15 @@
 from datetime import timedelta
 
-from container_app_conf import ConfigBase, EnvSource, YamlSource
+from container_app_conf import ConfigBase
 from container_app_conf.entry.bool import BoolConfigEntry
 from container_app_conf.entry.file import DirectoryConfigEntry
 from container_app_conf.entry.float import FloatConfigEntry
 from container_app_conf.entry.int import IntConfigEntry
 from container_app_conf.entry.list import ListConfigEntry
 from container_app_conf.entry.string import StringConfigEntry
 from container_app_conf.entry.timedelta import TimeDeltaConfigEntry
+from container_app_conf.source.env_source import EnvSource
+from container_app_conf.source.yaml_source import YamlSource
 
 NODE_MAIN = "py_image_dedup"
 

py_image_dedup/persistence/elasticsearchstorebackend.py

@@ -1,4 +1,5 @@
 import logging
+import time
 
 import requests
 from elasticsearch import Elasticsearch
@@ -40,8 +41,12 @@ def __init__(self,
         self.host = self.DEFAULT_DATABASE_HOST if host is None else host
         self.port = self.DEFAULT_DATABASE_PORT if port is None else port
 
+        detected_version = None
+        while detected_version is None:
+            time.sleep(2)
+            detected_version = self._detect_db_version()
+
         self._el_version = el_version
-        detected_version = self._detect_db_version()
         if self._el_version is not None and detected_version is not None and self._el_version != detected_version:
             raise AssertionError(
                 "Detected database version ({}) does not match expected version ({})".format(detected_version,
@@ -76,6 +81,7 @@ def __init__(self,
                     {'host': self.host, 'port': self.port}
                 ]
             ),
+            el_version=self._el_version,
             index=self._el_index,
             doc_type=self._el_doctype,
             distance_cutoff=max_dist

requirements.txt

@@ -1,9 +1,10 @@
 setuptools
-container-app-conf>=3.0.0
+pip>=19.2.3
+container-app-conf>=3.0.2
 requests
 scipy
 numpy
-image_match
+image_match @ git+https://github.com/markusressel/[email protected]#egg=image_match
 elasticsearch>=6
 tabulate
 tqdm

setup.py

@@ -73,7 +73,7 @@ def read_requirements_file(file_name: str):
     tests_require=test_requirements(),
     entry_points={
         'console_scripts': [
-            'py-image-dedup=py_image_dedup.cli.Main:cli'
+            'py-image-dedup = py_image_dedup.cli:cli'
         ]
     }
 )