python-samples.sh

#!/usr/bin/env bash
# This is python-samples.sh within https://github.com/wilsonmar/python-samples
# run to install and update external Python packages (dependencies) needed by "import" statements in the code.
# (to avoid ModuleNotFoundError: No module named ... )

THIS_PROGRAM="$0"
SCRIPT_VERSION="v0.0.13"

# After you obtain a Terminal (console) in your environment,
# cd to folder, copy this line and paste in the terminal (without the # character):
   # bash -c "$(curl -fsSL https://raw.githubusercontent.com/wilsonmar/python-samples/master/bash/python-samples.sh)" -v -i

# SECTION 1. Establish shell file run environment
# SECTION 2. Install utilities 
# SECTION 3. Obtain a copy of the repo holding source code and sample files
# SECTION 4. Create a virtual environment to isolate package dependency versions
# SECTION 5. Install pip packages in the enviornment
# SECTION 6. Install and run scans
# SECTION 7. Run Python interpreter on app code (python-samples.py)

# shellcheck disable=SC2001 # See if you can use ${variable//search/replace} instead.
# shellcheck disable=SC1090 # Can't follow non-constant source. Use a directive to specify location.

# ----------------------------------------------------

# SECTION 1. Establish shell file run enviornment

### Set ANSI color variables (based on aws_code_deploy.sh): 
bold="\e[1m"
dim="\e[2m"
# shellcheck disable=SC2034 # ... appears unused. Verify use (or export if used externally).
underline="\e[4m"
# shellcheck disable=SC2034 # ... appears unused. Verify use (or export if used externally).
blink="\e[5m"
reset="\e[0m"
red="\e[31m"
green="\e[32m"
# shellcheck disable=SC2034 # ... appears unused. Verify use (or export if used externally).
blue="\e[34m"
cyan="\e[36m"

h2() { if [ "${RUN_QUIET}" = false ]; then    # heading
   printf "\n${bold}\e[33m\u2665 %s${reset}\n" "$(echo "$@" | sed '/./,$!d')"
   fi
}
info() {   # output on every run
   printf "${dim}\n➜ %s${reset}\n" "$(echo "$@" | sed '/./,$!d')"
}
note() { if [ "${RUN_VERBOSE}" = true ]; then
   printf "\n${bold}${cyan} ${reset} ${cyan}%s${reset}" "$(echo "$@" | sed '/./,$!d')"
   printf "\n"
   fi
}
success() {
   printf "\n${green}✔ %s${reset}\n" "$(echo "$@" | sed '/./,$!d')"
}
error() {    # ☓
   printf "\n${red}${bold}✖ %s${reset}\n" "$(echo "$@" | sed '/./,$!d')"
}
warning() {  # ☞ or ☛
   printf "\n${cyan}☞ %s${reset}\n" "$(echo "$@" | sed '/./,$!d')"
}
fatal() {   # Skull: ☠  # Star: ★ ★ U+02606  # Toxic: ☢
   printf "\n${red}☢  %s${reset}\n" "$(echo "$@" | sed '/./,$!d')"
}

RUN_VIRTUALENV=false
RUN_PIPENV=false

run_pytest=false
run_gitleaks=false
run_flake8=false
run_autopep8=false
run_pylint=false
run_bandit=true

# if desired folder is not there, create it:
# cd to desired folder:
# cd project



# SECTION 2. Install utilities  ################################

# see sample.sh
#    1. XCode Command Utilities                   - see https://wilsonmar.github.io/xcode
#    2. Homebrew which installs MacOS utilities:  - see https://wilsonmar.github.io/homebrew
#    3. Homebrew packages: jq, gpg, etc.
#    4. Python IDE (PyCharm or VSCode or Cloud9)  - see https://wilsonmar.github.io/text-editors
#    5. configure IDE plug-ins
#    6. Miniconda virtual environment             - see https://wilsonmar.github.io/python-install
#    7. Python within Miniconda
#    8. The latest git client                     - see https://wilsonmar.github.io/git-install
#    9. AWS CLI                                   - see https://wilsonmar.github.io/aws-cli
#    10. Google Chrome browser and add-ons


# SECTION 3. Obtain a copy of the repo holding source code and sample files

# TODO: Wrap logic around this:
# If repo is there, upgrade it?
    # git clone https://github.com/wilsonmar/python-samples
    # cd python-samples


# SECTION 4. Create a conda environment  ===============================

if [ "${RUN_VIRTUALENV}" = true ]; then  # -V  (not the default pipenv)

   # https://levipy.com/virtualenv-and-virtualenvwrapper-tutorial
   # to create isolated Python environments. https://docs.python.org/3/library/venv.html
   #pipenv install virtualenvwrapper

   my_venv_folder="venv"
   if [ -d ${my_venv_folder} ]; then   # venv folder already in the folder:
      note "Folder {my_venv_folder} being re-used ..."
   else
      # The venv module is included in the Python standard library installed.
      h2 "Make venv ${my_venv_folder} ..."  # venv is for Python3, virtualenv is for Python2
      python3 -m venv "${my_venv_folder}"
      if [ ! -d "${my_venv_folder}" ]; then   # venv folder already in the folder:
         echo "${my_venv_folder} folder not found"
         exit 1
      fi
   fi

   if [ ! -d ${my_venv_folder} ]; then
      h2 "Activate {my_venv_folder}"
      source "${my_venv_folder}/bin/activate"
      # "(venv)" should now appear above the cursor.
   else
      echo "${my_venv_folder} not found. Aborting."
      exit
   fi

   # To check if one a virtual environment is active, check whether the 
   # VIRTUAL_ENV environment variable is set to the path of the virtual environment. 
   echo ${VIRTUAL_ENV}  # /Users/wilson_mar/gmail_acct/python-samples/venv

   # FIXME: -bash: syntax error near unexpected token `('
   # base_prefix=getattr(sys, "base_prefix", None)
      # base_prefix='/usr/local/opt/python@3.9/Frameworks/Python.framework/Versions/3.9'
      # sys.prefix='/Users/wilson_mar/gmail_acct/python-samples/venv'
   # echo "${sys.prefix}"
   # if ${my_venv_folder} is found at the end of sys.prefix:


   """
      # TODO: Make sure venv is in .gitignore:
   RESPONSE="$( find "venv" -type d -name *"${GitHub_REPO_NAME}"* )"
   if [ -n "${RESPONSE}" ]; then  # found somethiNg:
      note "${RESPONSE}"
      if [ "${REMOVE_GITHUB_BEFORE}" = true ]; then  # -d 
          ...
      fi
   fi
   """


      h2 "source venv/bin/activate"
      # shellcheck disable=SC1091 # Not following: venv/bin/activate was not specified as input (see shellcheck -x).
      source venv/bin/activate

      # RESPONSE=$( python3 -c "import sys; print(sys.version)" )
      RESPONSE=$( python3 -c "import sys, os; is_conda = os.path.exists(os.path.join(sys.prefix, 'conda-meta'))" )
      h2 "Within (venv) Python3: "
      # echo "${RESPONSE}"
     
   if [ -f "requirements.txt" ]; then
      # Created by command pip freeze > requirements.txt previously.
      # see https://medium.com/@boscacci/why-and-how-to-make-a-requirements-txt-f329c685181e
      # Install the latest versions, which may not be backward-compatible:
      pip3 install -r requirements.txt
   fi

fi

if [ "${RUN_PIPENV}" = true ]; then  # -V  (not the default pipenv)

   h2 "Use Pipenv by default (not overrided by -Virtulenv)"
   # https://www.activestate.com/blog/how-to-build-a-ci-cd-pipeline-for-python/

   # pipenv commands: https://pipenv.kennethreitz.org/en/latest/cli/#cmdoption-pipenv-rm
   note "pipenv in $( pipenv --where )"
      # pipenv in /Users/wilson_mar/projects/python-samples
   # pipenv --venv  # no such option¶
   
   note "$( pipenv --venv || true )"

   #h2 "pipenv lock --clear to flush the pipenv cache"
   #pipenv lock --clear

   # If virtualenvs exists for repo, remove it:
   if [ -n "${WORKON_HOME}" ]; then  # found somethiNg:
      # Unless export PIPENV_VENV_IN_PROJECT=1 is defined in your .bashrc/.zshrc,
      # and export WORKON_HOME=~/.venvs overrides location,
      # pipenv stores virtualenvs globally with the name of the project’s root directory plus the hash of the full path to the project’s root,
      # so several can be generated.
      PIPENV_PATH="${WORKON_HOME}"
   else
      PIPENV_PATH="$HOME/.local/share/virtualenvs/"
   fi
   RESPONSE="$( find "${PIPENV_PATH}" -type d -name *"${GitHub_REPO_NAME}"* )"
   if [ -n "${RESPONSE}" ]; then  # found somethiNg:
      note "${RESPONSE}"
      if [ "${REMOVE_GITHUB_BEFORE}" = true ]; then  # -d 
         pipenv --rm
            # Removing virtualenv (/Users/wilson_mar/.local/share/virtualenvs/bash-8hDxYnPf)…
            # or "No virtualenv has been created for this project yet!  Aborted!

         # pipenv clean  # creates a virtualenv
            # uninistall all dev dependencies and their dependencies:

         pipenv_install   # 
      else
         h2 "TODO: pipenv using current virtualenv ..."
      fi
   else  # no env found, so ...
      h2 "Creating pipenv - no previous virtualenv ..."
      PYTHONPATH='.' pipenv run python main.py    
   fi 

fi  # RUN_VIRTUALENV


# SECTION 5. Install pip packages environment  ###################

# TODO: If requirements.txt exists, use it::
# else:

#    Python packages (dependencies) needed by "import" statements in the code

     # Put these in the requirements.txt file or run the following:

       # https://realpython.com/python-virtual-environments-a-primer/
       # BEWARE https://github.com/Homebrew/homebrew-core/issues/76621
       # -U specifies Update (re-install) when run after initial installation.

       pip install -U python-dotenv  # see https://pypi.org/project/python-dotenv/
       pip install -U argparse  # to parse parameters
       pip install -U textblob  # localizes text using API calls to Google Translate.
       python -m textblob.download_corpora
       pip install -U keyring
       pip install -U azure-identity  # the Azure Active Directory identity library
       pip install -U azure-keyvault-secrets  # the Key Vault secrets library
       pip install -U boto3  # to access AWS (Amazon Web Services)
       pip install -U hvac   # hvac-0.11.2
       pip install google-cloud-secret-manager  # 2.8.0-py2.py3-none-any.whl (94 kB)
       pip install -U pyjwt   # https://github.com/jpadilla/pyjwt & https://pyjwt.readthedocs.io/en/stable/
       pip install -U redis   # redis-4.0.2-py3-none-any.whl (119 kB)
       sudo pip3 install -U pytz   # installed pytz-2021.3

# SECTION 6. Install and run scans

    # See https://www.securecoding.com/blog/best-python-open-source-security-tools/
    # 1. Unit Testing
   if [ "${run_pytest}" = true ]; then
       pip install -U pytest
       # pytest
   fi
    # 2. GitLeaks
    if [ "${run_gitleaks}" = true ]; then
        pip install -U gitleaks
    fi
    # 3. See https://flake8.pycqa.org/en/4.0.1/user/error-codes.html
    if [ "${run_flake8}" = true ]; then
       pip install -U flake8
       # Run:
       flake8 api-sample.py --ignore E111,E114,E116,E201,E202,E203,E221,E225,E231,E265,E266,E303,E501,W291,W293
    fi
   # 4. Lint program code using https://www.python.org/dev/peps/pep-0008/ and https://github.com/hhatto/autopep8
    if [ "${run_autopep8}" = true ]; then
       pip install -U autopep8
       # Run:
       autopep8 --in-place --aggressive --aggressive api-sample.py
       # Return of nothing means you're good?
    fi
   # 5. pylint which automates https://pylint.org/ = https://google.github.io/styleguide/pyguide.html
    if [ "${run_pylint}" = true ]; then
       pip install -U pylint
       pylint api-sample.py
    fi
   # 6. Pyreverse: UML Diagrams for Python - https://www.logilab.org/blogentry/6883
   # 7. Bandit
    if [ "${run_bandit}" = true ]; then
       pip install -U bandit
    fi

# SECTION 7. Run Python interpreter on app code (api-sample.py)
python api-sample.py

# TODO: Check to see if file already there:
   # If not, copy sample to $HOME and stop

# SECTION 8. Load Vault Client Proxy


systemctl webapp-??? & 

# SECTION 9. Run python-samples.py

note "Running ..."
./python-samples.py