Merge pull request #7 from cycloidio/develop

Merge develop on master

Author: talset

README.md

@@ -27,25 +27,27 @@ This script use env vars configuration to run ansible playbook with ssh proxy on
   * `(ANSIBLE_PLAYBOOK_PATH)`: Path of the ansible playbook to run. Default: `ansible-playbook`.
   * `(DEBUG)`: Run in debug mode
 
-ec2.py vars:
+ec2 vars:
   * `(AWS_INVENTORY)`: If the Amazon EC2 dynamic inventory need to be used or no, can be eiter `true`, `false` or `auto`. `auto` checks if `AWS_ACCESS_KEY_ID` is set or not. Default: `auto`.
   * `(AWS_ACCESS_KEY_ID)`: Used by Amazon EC2 dynamic inventory
   * `(AWS_SECRET_ACCESS_KEY)`: Used by Amazon EC2 dynamic inventory
   * `(EC2_VPC_DESTINATION_VARIABLE)`: Can be either `ip_address` for public ip address or `private_ip_address`, see [ec2.ini](https://github.com/ansible/ansible/blob/devel/contrib/inventory/ec2.ini). Default: `private_ip_address`.
 
-azure_rm.py vars:
+azure_rm vars:
   * `(AZURE_INVENTORY)`: If the Azure dynamic inventory need to be used or no, can be eiter `true`, `false` or `auto`. `auto` checks if `AZURE_SUBSCRIPTION_ID` is set or not. Default: `auto`.
   * `(AZURE_SUBSCRIPTION_ID)`: Used by Azure dynamic inventory
   * `(AZURE_TENANT_ID)`: Used by Azure dynamic inventory
   * `(AZURE_CLIENT_ID)`: Used by Azure dynamic inventory
   * `(AZURE_SECRET)`: Used by Azure dynamic inventory
   * `(AZURE_USE_PRIVATE_IP)`: Can be either `True` or `False`, see [azure_rm.py](https://raw.githubusercontent.com/ansible/ansible/devel/contrib/inventory/azure_rm.py). Default: `True`.
+  * `(ANSIBLE_PLUGIN_AZURE_PLAIN_HOST_NAMES)`: By default this plugin will use globally unique host names. This option allows you to override that, and use the name that matches the old inventory script naming.. Default: `False`.
+  note: Ansible `azure_rm` plugin is used for ansible `>= 2.8` else `azure_rm.py` script will be used
 
 Example of pipeline configuration :
 
 **YAML anchors**
 
-```
+```YAML
 shared:
   - &run-ansible-from-bastion
     config:
@@ -66,7 +68,7 @@ shared:
 
 **usage**
 
-```
+```YAML
     - task: run-ansible
       <<: *run-ansible-from-bastion
       params:
@@ -98,7 +100,7 @@ Example of pipeline configuration :
 
 **YAML anchors**
 
-```
+```YAML
 shared:
   - &aws-ami-cleaner
     task: aws-ami-cleaner
@@ -136,7 +138,7 @@ Example of pipeline configuration :
 
 **YAML anchors**
 
-```
+```YAML
 shared:
   - &aws-ecr-cleaner
     task: aws-ecr-cleaner
@@ -178,7 +180,7 @@ Example of pipeline configuration :
 
 **YAML anchors**
 
-```
+```YAML
 shared:
   - &vault-approle-login
     task: vault-approle-login
@@ -215,7 +217,7 @@ This script use env vars configuration to merge stack and config for Cycloid.io.
 
 **YAML anchors**
 
-```
+```YAML
 shared:
   - &merge-stack-and-config
     platform: linux
@@ -234,7 +236,7 @@ shared:
 
 **usage**
 
-```
+```YAML
     - task: merge-stack-and-config
       config:
         <<: *merge-stack-and-config
@@ -252,6 +254,21 @@ shared:
 ```
 
 
+# Build and test a local image
+
+```bash
+export IMAGE_NAME="cycloid/cycloid-toolkit:develop"
+export PYTHON_VERSION=3
+export ANSIBLE_VERSION=2.8.*
+docker build -t $IMAGE_NAME --build-arg=PYTHON_VERSION="$PYTHON_VERSION" --build-arg=ANSIBLE_VERSION="$ANSIBLE_VERSION" .
+
+virtualenv -p python3 --clear .env
+source .env/bin/activate
+pip install unittest2 docker
+python tests.py -v
+```
+
+
 # Push new image tag
 
 Tags are currently based on ansible version installed in the docker image.

files/ansible/hosts-template/default.azure_rm.yml.template

@@ -48,6 +48,9 @@ hostvar_expressions:
   # if none are found, the first public IP address.
   ansible_host: $ANSIBLE_PLUGIN_AZURE_HOST
 
+# By default this plugin will use globally unique host names. This option allows you to override that, and use the name that matches the old inventory script naming.
+plain_host_names: $ANSIBLE_PLUGIN_AZURE_PLAIN_HOST_NAMES
+
 # places hosts in dynamically-created groups based on a variable value.
 keyed_groups:
 # places each host in a group named 'tag_(tag name)_(tag value)' for each tag on a VM.

scripts/ansible-runner

@@ -27,19 +27,21 @@ usage()
     echo '  * `(ANSIBLE_PLAYBOOK_PATH)`: Path of the ansible playbook to run. Default: `ansible-playbook`.'
     echo '  * `(DEBUG)`: Run in debug mode'
     echo ''
-    echo 'ec2.py vars:'
+    echo 'ec2 vars:'
     echo '  * `(AWS_INVENTORY)`: If the Amazon EC2 dynamic inventory need to be used or no, can be eiter `true`, `false` or `auto`. `auto` checks if `AWS_ACCESS_KEY_ID` is set or not. Default: `auto`.'
     echo '  * `(AWS_ACCESS_KEY_ID)`: Used by Amazon EC2 dynamic inventory'
     echo '  * `(AWS_SECRET_ACCESS_KEY)`: Used by Amazon EC2 dynamic inventory'
     echo '  * `(EC2_VPC_DESTINATION_VARIABLE)`: Can be either `ip_address` for public ip address or `private_ip_address`, see [ec2.ini](https://github.com/ansible/ansible/blob/devel/contrib/inventory/ec2.ini). Default: `private_ip_address`.'
     echo ''
-    echo 'azure_rm.py vars:'
+    echo 'azure_rm vars:'
     echo '  * `(AZURE_INVENTORY)`: If the Azure dynamic inventory need to be used or no, can be eiter `true`, `false` or `auto`. `auto` checks if `AZURE_SUBSCRIPTION_ID` is set or not. Default: `auto`.'
     echo '  * `(AZURE_SUBSCRIPTION_ID)`: Used by Azure dynamic inventory'
     echo '  * `(AZURE_TENANT_ID)`: Used by Azure dynamic inventory'
     echo '  * `(AZURE_CLIENT_ID)`: Used by Azure dynamic inventory'
     echo '  * `(AZURE_SECRET)`: Used by Azure dynamic inventory'
     echo '  * `(AZURE_USE_PRIVATE_IP)`: Can be either `True` or `False`, see [azure_rm.py](https://raw.githubusercontent.com/ansible/ansible/devel/contrib/inventory/azure_rm.py). Default: `True`.'
+    echo '  * `(ANSIBLE_PLUGIN_AZURE_PLAIN_HOST_NAMES)`: By default this plugin will use globally unique host names. This option allows you to override that, and use the name that matches the old inventory script naming.. Default: `False`.'
+    echo '  note: Ansible `azure_rm` plugin is used for ansible `>= 2.8` else `azure_rm.py` script will be used'
     echo ''
     exit 1
 }
@@ -66,12 +68,14 @@ export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-west-1}"
 export EC2_VPC_DESTINATION_VARIABLE="${EC2_VPC_DESTINATION_VARIABLE:-private_ip_address}"
 
 # Default envvars for azure_rm.py
+export DEFAULT_ANSIBLE_PLUGIN_AZURE_HOST="${DEFAULT_ANSIBLE_PLUGIN_AZURE_HOST:-"(public_dns_hostnames + public_ipv4_addresses + private_ipv4_addresses) | first"}"
+export DEFAULT_ANSIBLE_PLUGIN_AZURE_HOST_PRIVATE="${DEFAULT_ANSIBLE_PLUGIN_AZURE_HOST_PRIVATE:-"(private_ipv4_addresses + public_dns_hostnames + public_ipv4_addresses) | first"}"
 export AZURE_INVENTORY="${AZURE_INVENTORY:-auto}"
+# Make sure args work for Ansible azure rm and https://raw.githubusercontent.com/ansible/ansible/devel/contrib/inventory/azure_rm.py
+export AZURE_TENANT="${AZURE_TENANT:-$AZURE_TENANT_ID}"
 export AZURE_USE_PRIVATE_IP="${AZURE_USE_PRIVATE_IP:-True}"
-export ANSIBLE_PLUGIN_AZURE_HOST="(public_dns_hostnames + public_ipv4_addresses) | first"
-if [ "${AZURE_USE_PRIVATE_IP,,}" == "true" ]; then
-    export ANSIBLE_PLUGIN_AZURE_HOST="private_ipv4_addresses | first"
-fi
+export ANSIBLE_PLUGIN_AZURE_PLAIN_HOST_NAMES="${ANSIBLE_PLUGIN_AZURE_PLAIN_HOST_NAMES:-False}"
+export ANSIBLE_PLUGIN_AZURE_HOST="${ANSIBLE_PLUGIN_AZURE_HOST:-""}"
 
 # Keep compatibility with old naming of ssh key
 export SSH_PRIVATE_KEY="${SSH_PRIVATE_KEY:-$BASTION_PRIVATE_KEY}"
@@ -99,8 +103,15 @@ if [ "$AWS_INVENTORY" == "auto" ] && [ -n "$AWS_ACCESS_KEY_ID" ] || [ "${AWS_INV
   cp /etc/ansible/hosts-template/ec2.py /etc/ansible/hosts/
   EXTRA_ANSIBLE_ARGS="${EXTRA_ANSIBLE_ARGS} -i /etc/ansible/hosts/ec2.py"
 fi
+if [ -z "${ANSIBLE_PLUGIN_AZURE_HOST}" ]; then
+  if [ "${AZURE_USE_PRIVATE_IP,,}" == "true" ]; then
+      export ANSIBLE_PLUGIN_AZURE_HOST="${DEFAULT_ANSIBLE_PLUGIN_AZURE_HOST_PRIVATE}"
+  else
+      export ANSIBLE_PLUGIN_AZURE_HOST="${DEFAULT_ANSIBLE_PLUGIN_AZURE_HOST}"
+  fi
+fi
 if [ "$AZURE_INVENTORY" == "auto" ] && [ -n "$AZURE_SUBSCRIPTION_ID" ] || [ "${AZURE_INVENTORY,,}" == "true" ]; then
-  if (( $(echo "${ANSIBLE_VERSION%.*} >= 2.7" |bc -l) )); then
+  if (( $(echo "${ANSIBLE_VERSION%.*} >= 2.8" |bc -l) )); then
     # Render default.azure_rm.yml template from envvars
     envsubst < /etc/ansible/hosts-template/default.azure_rm.yml.template > /etc/ansible/hosts/default.azure_rm.yml
     EXTRA_ANSIBLE_ARGS="${EXTRA_ANSIBLE_ARGS} -i /etc/ansible/hosts/default.azure_rm.yml"

tests.py

@@ -19,7 +19,7 @@
 # virtualenv -p python3 --clear .env
 # source .env/bin/activate
 # pip install unittest2 docker
-# 
+#
 # python tests.py
 
 # Debug print command that you can use
@@ -314,7 +314,7 @@ def test_basic(self):
 
         r = self.drun(cmd="cat playbook/.vault-password")
         self.assertTrue(self.output_contains(r.output, '.*password'))
-    
+
 
     def test_extra_args(self):
         environment={
@@ -412,7 +412,7 @@ def test_azure_hosts_inventory(self):
             'AZURE_SUBSCRIPTION_ID': 'foo',
         }
         r = self.drun(cmd="/usr/bin/ansible-runner", environment=environment)
-        if float(self.ansible_version) >= 2.7:
+        if float(self.ansible_version) >= 2.8:
             self.assertTrue(self.output_contains(r.output, '.*ansible-playbook.*-i /etc/ansible/hosts/default.azure_rm.yml'))
         else:
             self.assertTrue(self.output_contains(r.output, '.*ansible-playbook.*-i /etc/ansible/hosts/azure_rm.py'))
@@ -421,14 +421,52 @@ def test_azure_hosts_inventory(self):
         # Azure dynamic inventory should be used as AZURE_INVENTORY=true even if AZURE_SUBSCRIPTION_ID is not present
         environment={
             'AZURE_INVENTORY': 'true',
+            'ANSIBLE_PLUGIN_AZURE_PLAIN_HOST_NAMES': 'true',
         }
         r = self.drun(cmd="/usr/bin/ansible-runner", environment=environment)
-        if float(self.ansible_version) >= 2.7:
+        if float(self.ansible_version) >= 2.8:
             self.assertTrue(self.output_contains(r.output, '.*ansible-playbook.*-i /etc/ansible/hosts/default.azure_rm.yml'))
+            r = self.drun(cmd="cat /etc/ansible/hosts/default.azure_rm.yml")
+            self.assertTrue(self.output_contains(r.output, '^plain_host_names:.*true'))
+            # default ANSIBLE_PLUGIN_AZURE_HOST
+            self.assertTrue(self.output_contains(r.output, '^\s*ansible_host:.*private_ipv4_addresses \+ public_dns_hostnames \+ public_ipv4_addresses'))
         else:
             self.assertTrue(self.output_contains(r.output, '.*ansible-playbook.*-i /etc/ansible/hosts/azure_rm.py'))
         self.assertEquals(r.exit_code, 0)
 
+        # Azure dynamic inventory configuration
+        if float(self.ansible_version) >= 2.8:
+            # Override ANSIBLE_PLUGIN_AZURE_HOST
+            environment={
+                'AZURE_INVENTORY': 'true',
+                'AZURE_USE_PRIVATE_IP': 'True',
+                'ANSIBLE_PLUGIN_AZURE_HOST': 'foo',
+                'DEFAULT_ANSIBLE_PLUGIN_AZURE_HOST_PRIVATE': 'bar',
+                'DEFAULT_ANSIBLE_PLUGIN_AZURE_HOST': 'bli',
+            }
+            self.drun(cmd="/usr/bin/ansible-runner", environment=environment)
+            r = self.drun(cmd="cat /etc/ansible/hosts/default.azure_rm.yml")
+            self.assertTrue(self.output_contains(r.output, '^\s*ansible_host:.*foo'))
+
+            # AZURE_USE_PRIVATE_IP true
+            environment={
+                'AZURE_INVENTORY': 'true',
+                'AZURE_USE_PRIVATE_IP': 'True',
+                'DEFAULT_ANSIBLE_PLUGIN_AZURE_HOST_PRIVATE': 'bar',
+                'DEFAULT_ANSIBLE_PLUGIN_AZURE_HOST': 'bli',
+            }
+            self.drun(cmd="/usr/bin/ansible-runner", environment=environment)
+            r = self.drun(cmd="cat /etc/ansible/hosts/default.azure_rm.yml")
+            self.assertTrue(self.output_contains(r.output, '^\s*ansible_host:.*bar'))
+
+            # AZURE_USE_PRIVATE_IP false
+            environment={
+                'AZURE_INVENTORY': 'true',
+            }
+            self.drun(cmd="/usr/bin/ansible-runner", environment=environment)
+            r = self.drun(cmd="cat /etc/ansible/hosts/default.azure_rm.yml")
+            self.assertTrue(self.output_contains(r.output, '^\s*ansible_host:.*bli'))
+
         # Azure dynamic inventory should not be used as AZURE_INVENTORY=false even if AZURE_SUBSCRIPTION_ID is present
         environment={
             'AZURE_INVENTORY': 'false',
@@ -447,11 +485,12 @@ def test_ec2_and_azure_hosts_inventory(self):
             'AWS_ACCESS_KEY_ID': 'bar',
         }
         r = self.drun(cmd="/usr/bin/ansible-runner", environment=environment)
-        if float(self.ansible_version) >= 2.7:
+        if float(self.ansible_version) >= 2.8:
             self.assertTrue(self.output_contains(r.output, '.*ansible-playbook.*-i /etc/ansible/hosts/ec2.py.*-i /etc/ansible/hosts/default.azure_rm.yml'))
         else:
             self.assertTrue(self.output_contains(r.output, '.*ansible-playbook.*-i /etc/ansible/hosts/ec2.py.*-i /etc/ansible/hosts/azure_rm.py'))
         self.assertEquals(r.exit_code, 0)
 
+
 if __name__ == '__main__':
     unittest.main()