Programming notes

Python notes

See pyjsta package

Installation + Setup

Install anaconda from http://continuum.io/downloads to the default location (/anaconda)
Set the PYTHONSTARTUP environment variable to a py file, this will be run on startup

Environments

Anaconda

export PATH=/home/<user>/anaconda3/bin:$PATH
# conda env create -f environment.yml
# conda env update -f environment.yml
source activate foo

Sample environment yml file

name: foo
channels:
- conda-forge
dependencies:
        - python>3
        - pandas        
        - pytest
        - black
        - pip
        - pip:
                - somethingonpip
                - git+https://github.com/foo/somerepo.git

Remove environment

conda env remove -n foo

Virtual environments (venv)

# create
python -m venv /path/to/new/virtual/environment

# activate
source /path/to/new/virtual/environment/bin/activate

# deactivate
deactivate

Installing packages

  # from pypi
  pip install foo

  # from GH via SSH
  pip install git+ssh://git@github.com/user/repo.git

  # from GH branch via SSH
  pip install git+ssh://git@github.com/user/repo.git@branch

  # over proxy
  pip install foo --proxy http://myserver.com:9090

pytest

Run a specific test file

python -m pytest tests/my_test_file.py

Python command line operations

Execute string as python code

python -c "print(1+1)"

Create a GeoDataFrame row-wise from scratch

import pandas as pd
import geopandas as gpd


def create_gpd(x, crs=4326):
    return gpd.GeoDataFrame(
        x,
        geometry=gpd.points_from_xy([x["longitude"]], [x["latitude"]], crs=crs),
        index=[0],
    )

# fmt: off
test = [
    {"longitude": -140, "latitude": -65, "id": 0},
    {"longitude": -139, "latitude": -64, "id": 1},
    {"longitude": -138, "latitude": -63, "id": 2},
]
# fmt: on

pd.concat([create_gpd(x) for x in test])

Write a markdown table from pandas Dataframe

import pandas as pd
from tabulate import tabulate

table = pd.DataFrame([["Sun", 696000, 1989100000], ["Earth", 6371, 5973.6],
         ["Moon", 1737, 73.5], ["Mars", 3390, 641.85]])
mdtable = tabulate(table, tablefmt="simple", headers=["a", "b", "c"])
with open('mdtable.md', 'w') as f:
    f.write(mdtable)

echo \\\\pagenumbering{gobble}| cat - mdtable.md > temp && mv temp mdtable.md
pandoc mdtable.md -V fontsize=14pt -o $@
pdfcrop $@ $@

Save an arbitrary object as a pickle file

import pickle

x = 1
pickle.dump(x, open("x.pkl", "wb"))
x = pickle.load(open("x.pkl", "rb"))

Generate random strings for ids etc.

import secrets

secrets.token_hex(5)

Progress bar with token message

import time
import tqdm

for i in (pbar := tqdm.tqdm(range(100))):
    pbar.set_description(f"Processing {i}")
    time.sleep(1)

Ignoring a clause for coverage purposes

else: # pragma: no cover

Ignoring an import error in pylance

import package # type: ignore

Python Syntax

list files in a directory

import glob
glob.glob("a_folder/*")

return the “class” of an object

type(foo_object)

list the column names of a pandas object

list(pandas_object)

return valid methods of a given object

dir("blah") # gives list of relevant methods (e.g., blah = str)

duplicating arrays

a = [1,2,3]
b = a       # completely linked
c = a*1     # duplicate
a is b
a is not c

adding to lists

x = [2, -5, 3, 1, -3]
x.append(3)
x.extend([3,4])

dictionaries

x = {"name": "karl", "age": "really_old", "shoe_size": 8}
x["shoe_size"]
x.keys()
x.values()
list(x.keys())
list(x.values())

joining list elements into a string

a bit odd to me: separator.join(list)

x = ["a", "b", "c"]
",".join(x)

defining null values

x = None

loop

for i in range(10):
    print(i)

numpy

x = numpy.array([[1,2],[3,4],[5,6],[7,8]])
numpy.shape(x) # tuple (immutable) with (n_rows, n_cols)
x[0,0]
x[:,1]
x[2,:]

matplotlib

import matplotlib.pyplot as plt
plt.figure()
x = [xv+1 for xv in range(6)]
y = [xv**2 for xv in x]
plt.xlabel("X")
plt.ylabel("Y")
plt.title("first matplotlib plot")
plt.plot(x, y)
plt.show()

Various by python (v3.3) stuff while I learned the langugage that was really useful to me

google course on python: https://developers.google.com/edu/python/
software carpentry bootcamp: http://software-carpentry.org/v4/python/index.html

loops

for x in range(1, 10):
  print(x, end="")
print()

for x in range(9, 4, -1):
  print(x, end="")
print()

for x in range(0, 21, 5):
  print(x, end=" ")
print()

for i in range(1,6):
  print("%d^2 = %d" % (i, i**2))

i = 1
while i <= 5:
  print("%d^2 = %d" % (i, i**2))
  i += 1

string methods

print('Length of "This is a test"', end="")
print(len('This is a test'))

print('This is a test'.lower())

print('This is a test'.upper())

print('This is a test'.swapcase())

reverse (“extended slice syntax”: begin:end:step)

print("This is a test"[::-1])

string manipulation & reg ex

import re
### sub just the first
print(re.sub('bar', 'foo', 'foobarfoobar', count=1))

### split on whitespace
print("Blah blah blah. ".split())

### arrays (lists)
arr = [1, "test", 2, 3, 4]
for x in arr:
  print(str(x) + "X ", end="")
print()

### formatted print
for x in arr:
  print("%sX " % x, end="")
print()

### joining with .format
"this is a {type}".format(type="string")

map

x = list(map(lambda x:x+1, range(6)))
print(x)

list comprehension

x = [y+1 for y in range(6)]
print(x)

y = list(map(lambda x:x+2, range(6)))
print(' '.join(map(str, y)))

ranges

x = list(range(6))
y = list(range(1, 7))
z = list(range(3, 50, 5))
print(x, y, z)

other array methods

x = list(range(1,6))
y = [2, 4, 1]
print(x+y)
print(":".join(map(str, x+y)))

aliasing

x = list(range(1,6))
y = [2, 4, 1]
z = x # aliased
zz = list(x) # a copy
id(x) == id(z)  # True
id(x) != id(zz) # True
for yy in y:
  if yy in x: x.remove(yy)
print(x, z, zz)

print(3 in x)
print(7 in x)
print(x[0]) # first element
print(x[-1]) # last element

z = range(5, 9)
print(z[-2:]) # a range
z = list(z)
print(z[-2:]) # now a list
zz = z.reverse() # doesn't return
print(z, zz) # zz = None
zz = reversed(z)
print(zz) # an iterator
zz = list(reversed(z))
print(zz) # a list

hashes (hash is called a ‘dict’)

x = {"a" : 1, "b" : 2, "c" : 3}
print(x['a'])
for (value,key) in x.items()
  print(key, ' -> ', value)
print(list(x.keys()))
x.pop("a")
print(x)

x = {"a" : 1, "b" : 2, "c" : 3}
z = list(x.keys())  # need list() since I'll be modifying the keys in place
for key in z: # "for key in x:" would work if I weren't modifying the keys in place
  if x[key] == 2:
    x.pop(key)
print(x)

alternatively

x = {"a" : 1, "b" : 2, "c" : 3}
z = [key for key in x.keys() if x[key] == 2]
z = [key for key in x if x[key] == 2] # equivalently
for key in z:
  x.pop(key)
print(x)

x = {"a":1, "b":2}
x['d'] = x['d']+1 if 'd' in x else 1

slices of arrays, negative index to start from end

a = list(range(2, 13, 2))
print(a[1:3])
print(a[-1])
print(a[-3:-2])
print(a[-3])
print(a[-3:-1])

conversion between classes

int("5")        # to integer
float("6")      # to float
str(252.3)      # to string

a bit of text manipulation

text = '''We may at once admit that any inference from the particular
to the general must be attended with some degree of uncertainty,
but this is not the same as to admit that such inference cannot
be absolutely rigorous, for the nature and degree of the uncertainty
may itself be capable of rigorous expression.'''
stopwords = 'the a by on for of are with just but and to my in I has some'.lower().split()
words = text.lower().split()
keywords = [word for word in words if word not in stopwords]
print(' '.join(keywords))
print("no. char  =", len(' '.join(keywords)))
print("no. words =", len(keywords))

playing with map

n = 8
counts = map(lambda x: 0, range(n))
print(' '.join(map(str, counts)))
import random
x = map(lambda z: random.randint(1,8), range(1000))
counts = []
for i in range(1,9):
  counts.append( sum(z==i for z in y) )
print(' '.join(map(str, counts)))

looping over hashes (also sorting)

words = '''We may at once admit that any inference from the particular to the general
must be attended with some degree of uncertainty, but this is not the same as to
admit that such inference cannot be absolutely rigorous, for the nature and
degree of the uncertainty may itself be capable of rigorous expression.'''.split()
import re
words = list(map(lambda word: re.sub(r'[,\.]', '', word), words))
wordcount = {}
for word in words:
  wordcount[word] = wordcount[word]+1 if word in wordcount else 1

sort by word length

sorted(wordcount.keys(), key=len)

sort by count

sorted(wordcount.keys(), key=lambda x: wordcount[x])

by count then word length

sorted(wordcount.keys(), key=lambda x: [wordcount[x], len(x)])

by word length then count

sorted(wordcount.keys(), key=lambda x: [len(x), wordcount[x]])

by count then word length, but reversed

sorted(wordcount.keys(), key=lambda x: [wordcount[x], len(x)], reverse=True)

using a function

def count_and_length (a):
  return [wordcount[a], len(a)]
sorted(wordcount.keys(), key=count_and_length)

regex

import re
if not re.search(r'AM', 'am'):
  print('ok 1')
if re.search(r'(?i)AM', 'am'):
  print('ok 2')
if re.search(r'AM', 'am', re.IGNORECASE):
  print('ok 3')
multi = 'blah a number of special\nAll of these are'
if re.search(r'\Ablah', multi):
  print('ok 4')
if not re.search(r'\AAll', multi):
  print('ok 5')
if re.search(r'^blah', multi):
  print('ok 6')
if not re.search(r'^All', multi):
  print('ok 7')
if re.search(r'^A', multi, re.MULTILINE):
  print('ok 8')
if not re.search(r'special\Z', multi):
  print('ok 9')
if re.search(r'special$', multi, re.MULTILINE):
  print('ok 10')
if re.search(r'are\Z', multi, re.MULTILINE):
  print('ok 11')
if re.search(r'are\Z', multi):
  print('ok 12')
if not re.search(r'special$', multi):
  print('ok 13')
if re.search(r'special$', multi, re.MULTILINE):
  print('ok 14')
if re.search(r'are$', multi):
  print('ok 15')
if not re.search(r'blah.*are', multi):
  print('ok 16')
if re.search(r'blah.*are', multi, re.DOTALL):
  print('ok 17')
x = 'Today is 11/26/2013, while tomorrow is 11/27/2013.'
z = re.search(r'(\d+)/(\d+)/(\d+)', x)
if z:
  print('Month = %s, day = %s, year = %s' % (z.group(1), z.group(2), z.group(3)))
zz = re.findall(r'(\d+)/(\d+)/(\d+)', x)
if zz:
  print('Month = %s, day = %s, year = %s' % (zz[0][0], zz[0][1], zz[0][2]))
if len(zz) > 1:
  print('Month = %s, day = %s, year = %s' % (zz[1][0], zz[1][1], zz[1][2]))