python

pandas replace nan by ""

df = df.fillna("") # vervang alle nan's door "" in het hele dataframe

modify clipboard: for example escape all lines on the clipboard

import pyperclip

text = pyperclip.paste()
updated_text = "\n".join([line.lower() for line in text.splitlines()])

pyperclip.copy(updated_text)

csv.writer to write lut

for img in row["images"]:
  images_by_vh_id[img["image_id_vh"]] = img["image_name"]

with open("data/images_by_vh_id.csv", mode='w', newline='') as file:
  writer = csv.writer(file)
  writer.writerow(['image_id_vh', 'image_name'])
  writer.writerows(images_by_vh_id.items())

get inventarisnummer from filename

match = re.search(r'(?<=NL-UtHUA_)(.*)(?=_.*)', filename)
if match:
  nummer = match.group(1)

csv.DictReader from URL

with urllib.request.urlopen(url) as response:
  return csv.DictReader(response.read().decode('utf-8').splitlines())

csv.DictWriter (csv.writer)

with open("output.csv", mode='w', newline='') as file:
    writer = csv.DictWriter(file, fieldnames=["x", "y", "w", "h", "conf", "text"])
    writer.writeheader()
    writer.writerows(data)

download crops with iiif

import csv,os
import urllib
from urllib.request import urlretrieve
from tqdm import tqdm

aktes_per_scan = {row["nummer"]:int(row["aantal aktes"]) for row in csv.DictReader(open("data/aantal-aktes-op-scan-per-inv.csv"),delimiter=",") }
iiif = "https://files.transkribus.eu/iiif/2/"

def download(url, nummer, filename, suffix):
    folder = f"scans/{nummer}"
    filepath = folder + "/" + filename.replace(".jpg",f"_{suffix}.jpg")
    os.makedirs(folder, exist_ok=True)
    if not os.path.exists(filepath):
        print(url,filepath)
        urllib.request.urlretrieve(url, filepath)
    else:
        print(f"exists: {url}")

for row in tqdm(list(csv.DictReader(open("data/alle-scans-met-x-voor-overslaan.csv"),delimiter=",")),leave=True):
    id = row["iiif id"]
    nummer = row["nummer"]
    filename = row["filename"]
    aantal = aktes_per_scan[nummer]
    skip = row["overslaan"]!=""

    if skip or aantal==0:
        print(f"skip: {iiif}{id}/full/1600,/0/default.jpg")
        continue

    if aantal==4:
        download(f"{iiif}{id}/pct:0,0,50,50/1000,/0/default.jpg",nummer,filename,1)
        download(f"{iiif}{id}/pct:0,50,50,50/1000,/0/default.jpg",nummer,filename,2)
        download(f"{iiif}{id}/pct:50,0,100,50/1000,/0/default.jpg",nummer,filename,3)
        download(f"{iiif}{id}/pct:50,50,100,50/1000,/0/default.jpg",nummer,filename,4)

    if aantal==6:
        download(f"{iiif}{id}/pct:0,3,50,33/1000,/0/default.jpg",nummer,filename,1)
        download(f"{iiif}{id}/pct:0,33,50,33/1000,/0/default.jpg",nummer,filename,2)
        download(f"{iiif}{id}/pct:0,66,50,33/1000,/0/default.jpg",nummer,filename,3)
        download(f"{iiif}{id}/pct:50,3,50,33/1000,/0/default.jpg",nummer,filename,4)
        download(f"{iiif}{id}/pct:50,33,50,33/1000,/0/default.jpg",nummer,filename,5)
        download(f"{iiif}{id}/pct:50,66,50,33/1000,/0/default.jpg",nummer,filename,6)

    tqdm.write("")

mooi voorbeeld van argparse

• https://python-sounddevice.readthedocs.io/en/0.3.14/_downloads/49f259d60e5b087b6e1e380bbfddc3da/rec_unlimited.py

mkdir / makedirs

os.makedirs(output_dir, exist_ok=True)

create query for a lookup table from >1000 GUIDs with many OR statements

print ("select guid,id from A where 1=0")
for row in csv.DictReader(open("data/spreadsheet.csv",encoding="utf-8-sig"),delimiter=";"):
  guid = row['Guid']        
  print(f"OR guid='{guid}'")
print(";")

update query from CSV

import csv

with open("result.sql","w") as output_file:

    for row in csv.DictReader(open("input.csv",encoding="utf-8-sig"),delimiter=";"):
        id = row['id']

        beginjaar = row['Beginjaar']
        eindjaar = row['Eindjaar']
        openbaar_vanaf = row['Openbaar vanaf']

        sql=f"update ... set beginjaar='{beginjaar}', eindjaar='{eindjaar}', openbaar_vanaf='{openbaar_vanaf}' where id={id};"
        print(sql,file=output_file)

update query with subselect from excel sheet

import pandas as pd

data = pd.read_excel('Regesten toegang 218-1.xlsx')

for index, row in data.iterrows():
    guid = row['guid']
    waarde = row['Datering (oorspr)'].replace("'", "''")

    sql = f"""
UPDATE ... awe
SET awe.WAARDE = '{waarde}'
WHERE awe.fvd_id = 3859 AND awe.ahd_id IN (
    SELECT reg.id
    FROM ... reg
    WHERE reg.guid = '{guid}'
);
"""
    print(sql)

Error: WARNING: pip is configured with locations that require TLS/SSL, however the ssl module in Python is not available.

brew --prefix openssl@1.1

repeat items in list

colors = [akte_color]*3 + [overledene_color]*5
#result: ['#f8e5d0', '#f8e5d0', '#f8e5d0', '#d2e2f1', '#d2e2f1', '#d2e2f1', '#d2e2f1', '#d2e2f1']

write unique values from one column to a textfile

print("\n".join(sorted(set(item["NTNI"] for item in items))), file=open("data/ntni.txt","w"))

use Flask to render a CSV as a table using a template

import csv
from flask import Flask, render_template

app = Flask(__name__,template_folder="./")

def process_csv(csv_file):
    data = []
    with open(csv_file, newline='', encoding='utf-8') as csvfile:
        return list(csv.DictReader(csvfile))

@app.route('/')
def index():
    csv_file = 'input.csv'  # Replace with your CSV file path
    data = process_csv(csv_file)
    return render_template('template.html', data=data)

if __name__ == '__main__':
    app.run(debug=True)

csv to excel using pandas with filter and groupby

import pandas as pd

df = pd.read_csv('input.csv')
df = df[~df['flexveld'].isin(['Aangemaakt door', 'Aangemaakt op', 'Gewijzigd door', 'Gewijzigd op'])]

for name, group in df.groupby('aet_omschrijving'):
    writer = pd.ExcelWriter(f'output/{name}.xlsx', engine='xlsxwriter')
    group.to_excel(writer, index=False, sheet_name='Sheet1')
    workbook = writer.book
    worksheet = writer.sheets['Sheet1']
    worksheet.set_column('A:B', 30)
    (max_row, max_col) = df.shape
    column_settings = [{'header': column} for column in df.columns]
    writer.sheets['Sheet1'].add_table(0, 0, max_row, max_col - 1, {'columns': column_settings})
    writer.close()

merge two csv's

ik heb twee CSV's met de volgende headers:

1) org_row,filename,straatnaam,afbeelding,oude straatnaam 2) "uuid","name","width","height","link"

Ik wil graag een nieuwe CSV met de onderstaande velden waarbij de sleutelkolommen 'filename' en 'name' gebruikt kunnen worden om te matchen 2) "uuid","name","width","height","link","straatnaam"

response from ChatGPT 3.5. In 1x goed!

import csv

def merge_csv(csv1_path, csv2_path, output_path):
    data1 = {}
    with open(csv1_path, 'r') as csv1_file:
        reader1 = csv.DictReader(csv1_file)
        for row in reader1:
            data1[row['filename']] = row['straatnaam']

    with open(output_path, 'w', newline='') as output_file:
        fieldnames = ["uuid", "name", "width", "height", "link", "straatnaam"]
        writer = csv.DictWriter(output_file, fieldnames=fieldnames)
        writer.writeheader()

        with open(csv2_path, 'r') as csv2_file:
            reader2 = csv.DictReader(csv2_file)
            for row in reader2:
                street_name = data1.get(row['name'], '')
                row['straatnaam'] = street_name
                writer.writerow(row)

if __name__ == "__main__":
    csv1_path = "bestand1.csv"
    csv2_path = "bestand2.csv"
    output_path = "samengevoegd.csv"
    merge_csv(csv1_path, csv2_path, output_path)

format turtle file (or convert any rdf to formatted turtle)

from rdflib import Graph
g = Graph()
g.parse("INPUT.ttl")
print(g.serialize(format='ttl',encoding="utf-8").decode(), file=open("OUTPUT.ttl","w"))

upload file via requests

def upload(url,token,filename):
    headers = { "Authorization": f"Bearer {token}" }
    data = { "mergeGraphs": "true" }
    files = { "file": (os.path.basename(filename), open(filename,'rb'), 'application/zip') }
    response = requests.post(url, headers=headers, data=data, files=files)
    return response.json()

streaming subprocess data through flask

https://stackoverflow.com/a/15354403/6068235

import flask
import subprocess

app = flask.Flask(__name__)

@app.route('/yield')
def index():
    def inner():
        proc = subprocess.Popen(
            ['sh upload.sh'],             #call something with a lot of output so we can see it
            shell=True,
            stdout=subprocess.PIPE
        )

        for line in iter(proc.stdout.readline,''):
            yield f"{line.decode('utf8')}<br/>\n"

    return flask.Response(inner(), mimetype='text/html')  # text/html is required for most browsers to show th$

app.run(debug=True, port=8080, host='0.0.0.0')

spacy

import spacy
text = open("X.txt").read()
nlp = spacy.load("nl_core_news_sm")
doc = nlp(text)
# print(doc.text)
for token in doc:
    print(token.text, token.pos_, token.dep_)

ip adres

import socket
hostname = socket.gethostname()
ip_adres = socket.gethostbyname(hostname)

filename without directory and without extension

os.path.splitext(os.path.basename(csv_path))[0]

flatten folder structure for jpg's and PageXML's:

#!/usr/bin/env python3

from pathlib import Path
import os,sys,re,shutil

input_base = "/Users/rick/Loghi (vervolg)/Resultaat fase 2/fase-2_output/"
output_base = "output/"

# toegang_inv_base = "702.2244"  # extra submap gemaakt voor alle deelbeschrijvingen
toegang_inv_base = "702.121"  # extra submap gemaakt voor alle deelbeschrijvingen

for img_path in Path(input_base + toegang_inv_base).rglob("*.jpg"):

    img_folder = os.path.dirname(img_path)
    img_filename = os.path.basename(img_path)
    toegang_inv = os.path.basename(img_folder)
    page_path = img_folder + "/page/" + img_filename.replace(".jpg",".xml")

    result = re.split(r'\.', toegang_inv, maxsplit=1) # let op maxsplit
    toegang, inv = result

    #JPGs
    Path(output_base + toegang_inv_base).mkdir(parents=True, exist_ok=True)
    shutil.copy2(img_path, output_base + toegang_inv_base)

    #pageXMLs
    Path(output_base + toegang_inv_base + "/page/" ).mkdir(parents=True, exist_ok=True)
    shutil.copy2(page_path, output_base + toegang_inv_base + "/page/")

read PageXML without metadata using regex

def get_pagexml_without_metadata(xml_file):
    return re.sub("<Metadata>.*?</Metadata>", "", open(xml_file).read(), flags=re.DOTALL)

tip van Stan

NLTK - Natural Language Toolkit

SMF SimpleMachines Forum Export / Backup to JSON

import mysql.connector, json

def get_column_names(db_name, table_name):
    query = f"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{db_name}' AND TABLE_NAME = '{table_name}'"
    cursor.execute(query)
    return [row["COLUMN_NAME"] for row in cursor]

db_name = 'your_db'
db_user = 'root'
db_pass = ''
conn = mysql.connector.connect(user=db_user, password=db_pass, host='127.0.0.1', database=db_name)
cursor = conn.cursor(dictionary=True)

board_columns = get_column_names(db_name,'smf_boards')
topic_columns = get_column_names(db_name,'smf_topics')
message_columns = get_column_names(db_name,'smf_messages')

cursor.execute("""
SELECT * FROM smf_messages m
JOIN smf_topics t ON t.id_topic = m.id_topic
JOIN smf_boards b ON b.id_board = t.id_board
ORDER BY b.id_board, t.id_topic, m.id_msg
""")

structure = {}

for row in cursor:
    board_id = row['id_board']
    topic_id = row['id_topic']

    # make board data
    board_data = {col: row[col] for col in board_columns}

    # make topic data
    topic_data = {col: row[col] for col in topic_columns}

    # add board toe if not exits
    if board_id not in structure:
        structure[board_id] = {'board_data': board_data, 'topics': {}}

    # add topic if not exists
    if topic_id not in structure[board_id]['topics']:
        structure[board_id]['topics'][topic_id] = {'topic_data': topic_data, 'messages': []}

    # make message data and add tot topics
    message_data = {col: row[col] for col in message_columns}
    structure[board_id]['topics'][topic_id]['messages'].append(message_data)

cursor.close()
conn.close()

json.dump(structure, open("result.json","w"), indent=2, ensure_ascii=False)

lookup hierarchy from id's in CSV file and output new CSV

#!/usr/bin/env python3
import csv

items = list(csv.DictReader(open("beschrijvingen-met-parent-guids.csv")))
items_by_guid = {}

#build lut
for row in items:
    row["parents"] = row["parents"].split(" ")
    items_by_guid[row["GUID"]] = row

#solve refs
for item in items_by_guid.values():
    item["parents"] = [ items_by_guid[p_guid] for p_guid in item["parents"] if not item==items_by_guid[p_guid]]

# als er een inleiding gevonden wordt direct onder een rubriek
# dan deze inleiding toevoegen aan de rubriek omschrijving
for item in items:
    if item["aet"]=="inl" and item["parents"][-1]["aet"]=="rub":
        item["parents"][-1]["omschr"] += f"\n{item['omschr']}"

# clean up
for item in items:
    item["omschr"] = item["omschr"].replace(">tr.",">getrouwd met: ")
    item["omschr"] = item["omschr"].replace("<BR>","\n")

header = ["guid","text"]
writer = csv.DictWriter(open("items-with-context.csv","w",encoding="utf-8"), fieldnames=header, delimiter=';', quoting=csv.QUOTE_ALL, dialect='excel')
writer.writeheader()

for item in items:
    writer.writerow({
        "guid": item["GUID"], 
        "text": "\n".join([p["omschr"] for p in item["parents"]] + [item["omschr"]])
    })

Count entities (in many json-files) grouped by entity_type

#!/usr/bin/env python3
import json
from pathlib import Path
from collections import defaultdict

entities_by_type = { }

for filename in list(Path(f"data/").rglob("*.json")):
    try:
       data = json.load(open(filename))
       for item in data["gpt-result"]:
            for value in data["gpt-result"][item]:
                value = str(value).strip()

                if not item in entities_by_type:
                    entities_by_type[item] = defaultdict(int)

                entities_by_type[item][value] += 1

    except Exception as e:
       print(e,filename)

# sort by count within entity_type
for t in entities_by_type:
   entities_by_type[t] = dict(sorted(entities_by_type[t].items(), key=lambda x:x[1], reverse=True))

json.dump(entities_by_type, open("entities.json","w"),indent=2,ensure_ascii=False)

Call GPT-4 API for Namd Entity Recognition (NER)

#!/usr/bin/env python3

import json,csv,os,openai,sys

openai.api_key = os.getenv("OPENAI_API_KEY")

for row in list(csv.DictReader(open("beschrijvingen-platte-lijst.csv")))[0:1000]:
    filename = "data/" + row["GUID"] + ".json"
    if os.path.exists(filename):
        continue

    response = openai.ChatCompletion.create(
        # model="gpt-3.5-turbo",
        model="gpt-4-1106-preview",
        response_format={ "type": "json_object" },
        messages=[ {"role": "system", "content": "You can recognize named-entities and return these in JSON as strings grouped by entity-type: { 'persons':[], 'organisations':[], 'locations':[], 'topics':[], 'events':[], 'dates':[]"},
                   {"role": "user", "content": row["context"] } ]
    )

    try:
        if response:
            response["text"] = json.loads(response["choices"][0]["message"]["content"])
    except Exception as e:
        print(e,response)
        pass

    try:
        json.dump({"item":row, "gpt-result":response["text"]},open(filename,"w"), indent=2, ensure_ascii=False)
        print(filename, response["text"])
    except Exception as e:
        print(e,row["GUID"],response)

    print(filename)

escape URI part

def escape_URI_part(s):
  # spreadsheet: [–’&|,\.() ""$/':;]"; "-") ;"-+";"-"); "[.-]$"; ""))
  s = re.sub(r"[_–’+?&=|,\.() \"$/']", "-", s) # replace different characters by a dash
  s = re.sub(r"-+", "-", s) # replace multiple dashes by 1 dash
  s = re.sub(r"[^a-zA-Z0-9\-]", "", s) # strip anything else now that is not a alpha or numeric character or a dash
  s = re.sub(r"^-|-$", "", s) # prevent starting or ending with . or -
  if len(s)==0:
    # raise ValueError("makeSafeURIPart results in empty string")
    # log.warning("makeSafeURIPart results in empty string")
    # fix this by replacing by 'x' for example
    import random
    s="unknown"+str(int(random.randint(10000, 99999)))
  return s.lower()

pretty print

from pprint import pprint 
print(data)

get all pages

from bs4 import BeautifulSoup
import requests

url = "https://www.website.org/topics/"

while url:
    print(url)
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    url = soup.find("link", rel="next")
    if url:
        url = url.get("href")

get item by id+value in json array

find item with code=="FT"

item = next((item for item in data if item.get("code") == "FT"), None)

as function:

def get_item(items, key, value):
    return next((item for item in items if item.get(key) == value), None)

find multiple values:

def get_items(items, key, values):
    return [ next((item for item in items if item.get(key) == value), None) for value in values]

merge list with z_values with 2D nparray

z_values = [img[int(y), int(x), 2] for x, y in points]
points = np.column_stack((points[:, 0], points[:, 1], z_values))

shorter:

points = np.column_stack((points, img[points[:, 1].astype(int), points[:, 0].astype(int), 2]))

pretty print

from pprint import pprint
pprint(data)

merge files from 2 folders to an output folder

#!/usr/bin/env python3

from pathlib import Path
import os,shutil

alto_folder = "XX/input/"
alto_files_by_filename = {os.path.basename(path): path for path in Path(alto_folder).rglob("*.xml")}
image_filepaths = list(Path(f"1001/").rglob("*.jpg"))

for image_path in image_filepaths :
    # recreate folder structure from images
    image_folder = os.path.dirname(image_path)
    image_filename = os.path.basename(image_path)
    dst_folder =  Path("output/" + image_folder)
    dst_folder.mkdir(parents=True, exist_ok=True) 
    dst_image_path = os.path.join(dst_folder, image_filename)

    # copy images
    if not os.path.exists(dst_image_path):
        print("copy",image_path,"to",dst_image_path)
        shutil.copy2(image_path, dst_folder)

    # copy and rename alto files
    alto_filename = image_filename.replace(".jpg",".xml")
    alto_filepath = alto_files_by_filename.get(alto_filename)
    if alto_filepath: # skip non-existing alto files
        new_alto_filename = alto_filename.replace(".xml","_alto.xml")
        dst_alto_filepath = os.path.join(dst_folder, new_alto_filename)
        shutil.copy2(alto_filepath, dst_alto_filepath)

store filepaths (recursively) in a dict by filename

alto_files_by_filename = {os.path.basename(path): path for path in Path(alto_folder).rglob("*.xml")}

maak lookup table

lut_technieken = { item["INHOUD"]:item["ID"] for item in csv.DictReader(open("con_technieken.csv")) }

image areas in html to json

from bs4 import BeautifulSoup
import json

with open("index.html", "r", encoding="utf-8") as file:
    html = file.read()

soup = BeautifulSoup(html, 'html.parser')
area_tags = soup.find_all('area')

json_output = []

for area in area_tags:
    area_dict = {}
    area_dict['TITLE'] = area.get('title')
    area_dict['ID'] = area.get('id')
    area_dict['REL'] = area.get('rel')
    area_dict['HREF'] = area.get('href')
    area_dict['DATA_X_ABS'] = area.get('data-x-abs')
    area_dict['DATA_Y_ABS'] = area.get('data-y-abs')
    area_dict['TARGET'] = area.get('target')
    area_dict['CLASS'] = area.get('class')
    area_dict['SHAPE'] = area.get('shape')

    coords = area.get('coords')
    if coords:
        coords_list = list(map(int, coords.split(',')))
        area_dict['COORDS'] = coords_list

    json_output.append(area_dict)

json_str = json.dumps(json_output, indent=4)
print(json_str)

build exe file

pip install pyinstaller
pyinstaller --add-data 'data;data' .\YourScript.py   # with data folder

random integer

random.randint(1, 10)

draw centered fullscreen chessboard

import cv2
import numpy as np

image_width = 3840
image_height = 2400

img = np.zeros((image_height, image_width, 3), dtype=np.uint8)

square_size = image_height // 8
for y in range(8):
    for x in range(8):
         if (x + y) % 2 == 0:              
            x1 = (image_width//2-image_height//2) + x*square_size
            y1 = y*square_size
            x2 = x1+square_size
            y2 = y1+square_size
            img[y1:y2, x1:x2] = (255,255,255)

cv2.namedWindow('img', cv2.WINDOW_NORMAL)
cv2.setWindowProperty('img', cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
cv2.imshow('img', img)
cv2.waitKey(0)
cv2.destroyAllWindows()

opencv - mask, crop, channels, stretch, colorize with lut

import cv2
import numpy as np

palette_image = cv2.imread('palette256.png')
palette_lut = palette_image[0,:,:]
frame = cv2.imread("frame.png")
height,width = frame.shape[:2]
(cx,cy),r = (321, 279),226 # for frame.png
names="red","green","blue",
g=[202,129,185] # range start per channel
a,b,c = [None]*3, [None]*3, [None]*3 # maak 3 lege lists met ruimte voor 3 items

def stretch(img,minv,maxv):
    alpha = 255/(maxv-minv)
    return cv2.convertScaleAbs(img, alpha=alpha, beta = -minv*alpha)  

while(True):
    mask = np.zeros((height, width), dtype=np.uint8)
    cv2.circle(mask, (cx,cy), r, 255, -1)
    masked = cv2.bitwise_and(frame, frame, mask=mask)
    cropped = masked[cy-r:cy+r, cx-r:cx+r]
    h,w,_ = cropped.shape

    for i in range(3): # 3 channels
        a[i] = cropped[:,:,i]
        b[i] = stretch(a[i],g[i],255)
        c[i] = palette_lut[b[i].astype(int)]

        cv2.imshow(f"{names[i]}", a[i]); cv2.moveWindow(f"{names[i]}", i*w, 0)
        cv2.imshow(f"{names[i]}_stretched", b[i]); cv2.moveWindow(f"{names[i]}_stretched", i*w, h)
        cv2.imshow(f"{names[i]}_colored", c[i]); cv2.moveWindow(f"{names[i]}_colored", i*w, 2*h)

    key = cv2.waitKey(1) & 0xFF
    if key == 27:
        break
    elif key==ord('-'):
        radius -= 1;
    elif key==ord('='):
        radius += 1;
    elif key==ord('x'):
        center[0] += 1;
    elif key==ord('X'):
        center[0] -= 1;
    elif key==ord('y'):
        center[1] += 1;
    elif key==ord('Y'):
        center[1] -= 1;
    elif key==ord('r'):
        g[0] += 1;
    elif key==ord('R'):
        g[0] -= 1;
    elif key==ord('g'):
        g[1] += 1;
    elif key==ord('G'):
        g[1] -= 1;
    elif key==ord('b'):
        g[2] += 1;
    elif key==ord('B'):
        g[2] -= 1;
    elif key == ord(' '):
        for i in range(3):
            print(f"{names[i]}={g[i]}")
        print("center",center,"radius",radius)

cv2.destroyAllWindows()

various pip install libraries

pip install opencv-python

jinja2

#!/usr/bin/env python3

import json,re,sys
from jinja2 import Environment, FileSystemLoader

env = Environment(loader=FileSystemLoader("templates"))

def ref(id):
    return items_by_id[id] 

def guid_from_id(id):
    if not id:
        return ""
    return id[4:]

def dump(item):
    if not item:
        return ""
    return "#"+json.dumps(item,indent=2).replace("\n","\n#")

def escape_key_name(s): # for ninja2/Liquid templates
    s = re.sub(r"@","",s) # @id @type etc
    s = re.sub(r".*:","",s) # remove prefix:
    return s

def escape_key_names(item):
  try:
    result = { escape_key_name(x): v for x, v in item.items() } # renames keynames to use in template
  except Exception as e:
    print(item)
    sys.exit(1)
  return result

env.filters['guid'] = guid_from_id
env.filters['ref'] = ref
env.filters['dump'] = dump

data = json.load(open("data.json"))
items = data["@graph"][:4]
items = [escape_key_names(item) for item in items]
items_by_id = { item["id"]:item for item in items }

for item in items:
    # print(item)
    tmpl = env.get_template(item["type"].replace("aet:","")+".ttl")
    print(tmpl.render(**item))

copy 1000 random files from a csv

#!/usr/bin/env python3

import shutil,os
import pandas as pd

df = pd.read_csv("../test-omgekeerde-kaartjes/output.csv") 

# filter alle kaarten weg van waar niet zeker van is dat ze goed zijn
df = df[df['verkeerd_om'] == 'False'] 

# neem een random sample van 1000 stuks
# altijd dezelfde sample door de 'random_state=42'. 
# hierdoor hoeft de output map niet eerst leeg gemaakt te worden
# mocht je dat wel willen dan kan dat met # shutil.rmtree("output/")
random_items = df.sample(n=1000, random_state=42)

# kopiëer 1000 eerder gecropte bestanden ('crops/boven') naar de output/ map
for index,item in random_items.iterrows():
    input_path = item["filename"].replace("../","../crops/boven/")
    rel_path = item["filename"].replace("../","")
    output_folder = "output/" + os.path.dirname(rel_path)
    basename = os.path.basename(rel_path)

    # maak map aan indien nodig
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)

    # kopiëren indien nodig
    output_file_path = os.path.join(output_folder,basename)
    print(input_path," >>> ", output_folder)
    if not os.path.exists(output_file_path):
        shutil.copy2(input_path, output_file_path)

incremental update of sqlite database from CSV file

#!/usr/bin/env python3
import csv,sqlite3
from tqdm import tqdm
import argparse

parser=argparse.ArgumentParser()
parser.add_argument("--csv", help="input csv file with format: ID,GUID", required=True)
args=parser.parse_args()

csv_file = args.csv
db_file = 'database.db'
table_name = 'guids'

conn = sqlite3.connect(db_file)
cursor = conn.cursor()
cursor.execute(f"CREATE TABLE IF NOT EXISTS {table_name} (ID INTEGER, GUID TEXT)")
cursor.execute(f"CREATE INDEX IF NOT EXISTS idx_id ON {table_name} (ID)")
cursor.execute(f"CREATE INDEX IF NOT EXISTS idx_guid ON {table_name} (GUID)")

num_rows = sum(1 for _ in open(csv_file))

with open(csv_file, 'r') as file:
    csv_data = csv.reader(file)
    next(csv_data) # skip header
    for i, row in tqdm(enumerate(csv_data, start=1), total=num_rows):
        cursor.execute(f"REPLACE INTO {table_name} (ID, GUID) VALUES (?, ?)", (int(row[0]), row[1]))

conn.commit()
conn.close()

read csv with DictReader

rows = list(csv.DictReader(open("urls-lijst-van-picturae-verrijkt-met-sql.csv"), delimiter=","))

write (filtered) rows to Excel with pandas

#write result as Excel
excel_writer = pd.ExcelWriter("probleem-bestandskoppelingen.xlsx", engine='xlsxwriter')
df = pd.DataFrame(results)
df = df[df['info'] != 'ok'] # filter alles weg met 'ok' in de kolom 'info'
df.to_excel(excel_writer, sheet_name="Sheet1", startrow=1, header=False, index=False)
(max_row, max_col) = df.shape
sheet = excel_writer.sheets['Sheet1']
[sheet.set_column(i,i,w) for i,w in enumerate([20,10,50,10,12,50,10,10,50])] # set column widths
sheet.add_table(0, 0, max_row, max_col-1, {'columns': [{'header': col} for col in df.columns]})
excel_writer.close()

copy file and rebuild structure

def check(row): #check/process single file
    rel_path = re.sub(r"X:[\\/]yyy[\\/]","",row["path"]).replace("\\","/")
    abs_path = "/Volumes/xxx$/yyy/" + rel_path
    file_exists = os.path.exists(abs_path) and os.path.isfile(abs_path)
    if file_exists:
        # kopieer bestand naar lokaal en maak directories aan
        output_folder = "output/" + os.path.dirname(rel_path)
        basename = os.path.basename(rel_path)
        if not os.path.exists(output_folder):
            os.makedirs(output_folder)
        output_file_path = os.path.join(output_folder,basename)
        if not os.path.exists(output_file_path):
            shutil.copy2(abs_path, output_file_path)

parallel

from joblib import Parallel, delayed

def check(row):
  return row

results = Parallel(n_jobs=-1)(delayed(check)(row) for row in tqdm(rows))
print(results)

image check

from PIL import Image

# do image check
w = h = ""
if not file_exists:
    image_info = "bestand niet gevonden"
elif os.path.getsize(abs_path)==0:
    image_info = "bestand is leeg"
else:
    try:
        im = Image.open(output_file_path)            
        im.verify() #I perform also verify, don't know if he sees other types o defects
        im.close() #reload is necessary in my case
        im = Image.open(output_file_path) 
        im.transpose(Image.FLIP_LEFT_RIGHT)
        im.close()
        image_info = "ok"
        w, h = im.size
    except Exception as e: 
        image_info = f"fout in afbeelding: {e}"
        passass

crop images

#!/usr/bin/env python3

import sys,os
from tqdm import tqdm
from pathlib import Path
from PIL import Image

folder_names = tqdm(["a","b","c"])

for folder in folder_names:
folder_names.set_description(f"{folder}")
images = sorted(list(Path(f"../xx/{folder}/").rglob("*.jpg")))
file_names = tqdm(images,leave=False)

for image in file_names:
    filename = str(image)
    file_names.set_description(f"{filename}")

    img = Image.open(filename)
    w, h = img.size

    for naam,y1,y2 in [ ("boven",0,1500), ("onder", 1500,h)]:
    output_filename = filename.replace("..", naam)
    output_folder = os.path.dirname(output_filename)
    if not os.path.exists(output_filename):
        if y2>y1: 
        crop = img.crop((0, y1, w, y2))
        else:
        crop = img
        os.makedirs(output_folder, exist_ok=True)
        crop.save(output_filename)

load json

with open(input_file_path,"r") as file:    
  data = json.load(file)

ocr on Mac

pip3 install pyobjc-framework-Vision
pip3 install pyobjc-framework-Quartz
pip3 install wurlitzer

no subproces (not parallel):

import Quartz,Vision,json,csv
from Cocoa import NSURL
from Foundation import NSDictionary
from wurlitzer import pipes # needed to capture system-level stderr

def ocr(image_filename):
    input_url = NSURL.fileURLWithPath_(image_filename)
    with pipes() as (out, err):
        input_image = Quartz.CIImage.imageWithContentsOfURL_(input_url)
    (width,height) = input_image.extent().size
    vision_options = NSDictionary.dictionaryWithDictionary_({})
    vision_handler = Vision.VNImageRequestHandler.alloc().initWithCIImage_options_(
        input_image, vision_options
    )

    request = Vision.VNRecognizeTextRequest.alloc().init().autorelease()
    request.setRecognitionLevel_(Vision.VNRequestTextRecognitionLevelAccurate) #VNRequestTextRecognitionLevelFast
    request.setRecognitionLanguages_(["nl-NL"])
    error = vision_handler.performRequests_error_([request], None)

    results = []
    for item in request.results():
        bbox = item.boundingBox()
        w, h = bbox.size.width, bbox.size.height
        x, y = bbox.origin.x, bbox.origin.y
        results.append({
            "x":int(x*width),
            "y":int(height - y*height - h*height),
            "w":int(w*width),
            "h":int(h*height),
            "conf":item.confidence(),
            "text":item.text()
        })
    return results

in parallel:

#!/Applications/Xcode.app/Contents/Developer/usr/bin/python3

import config,json,os,subprocess,ocr2json
from tqdm import tqdm
from joblib import Parallel, delayed

import Quartz,Vision
from Cocoa import NSURL
from Foundation import NSDictionary
from wurlitzer import pipes # needed to capture system-level stderr

def ocr(image_filename):
    input_url = NSURL.fileURLWithPath_(image_filename)
    with pipes() as (out, err):
        input_image = Quartz.CIImage.imageWithContentsOfURL_(input_url)
    (width,height) = input_image.extent().size
    vision_options = NSDictionary.dictionaryWithDictionary_({})
    vision_handler = Vision.VNImageRequestHandler.alloc().initWithCIImage_options_(
        input_image, vision_options
    )

    request = Vision.VNRecognizeTextRequest.alloc().init().autorelease()
    request.setRecognitionLevel_(Vision.VNRequestTextRecognitionLevelAccurate) #VNRequestTextRecognitionLevelFast
    request.setRecognitionLanguages_(["nl-NL"])
    error = vision_handler.performRequests_error_([request], None)

    results = []
    for item in request.results():
        bbox = item.boundingBox()
        w, h = bbox.size.width, bbox.size.height
        x, y = bbox.origin.x, bbox.origin.y
        results.append({
            "x":int(x*width),
            "y":int(height - y*height - h*height),
            "w":int(w*width),
            "h":int(h*height),
            "conf":item.confidence(),
            "text":item.text()
        })
    return results

def do_ocr(image_file_path, progress):
    file_id = config.get_id_from_path(image_file_path)
    json_file_path = config.make_path(base_folder=config.DEEDS_OCR_JSON_FOLDER, id=file_id, suffix=".json")

    if not os.path.exists(json_file_path):
        data = ocr(str(image_file_path))
        if data and len(data)>0:
            json.dump(data, open(json_file_path,"w"), indent=2)
        print(json_file_path,str(int(progress*1000)/10.)+"%")

def run():
    print("run ocr_per_akte")

    if not os.path.exists(config.DEEDS_OCR_JSON_FOLDER):
        os.mkdir(config.DEEDS_OCR_JSON_FOLDER)

    file_paths = config.get_deeds_image_file_paths()
    results = Parallel(n_jobs=1, prefer="threads")(
      delayed(do_ocr)(image_file_path,i/len(file_paths))
      for i,image_file_path in enumerate(file_paths)
    )

if __name__ == '__main__':
    run()

show all sqlite tables

res = cursor.execute("SELECT name FROM sqlite_master")
result = res.fetchone()
print(result)

parse number sequence

#40630837,"1939","165-172 : ill."
#40630838,"1996","4-8 : ill., plgr., tek."
#40630839,"1996","10-13 : portr."
#40630843,"1996","18"
#40631828,"1969","41-49"
#40631829,"1969","51-60"
#40631836,"1969","99-119"
#40631846,"1970","68-70"
#40631847,"1970","93-102 : ill."
#40632861,"1931","38-40"
#40632864,"1931","42-44; 57-59"
#40633811,"1943","1-3"
#40633813,"1943","4-7"
#40633814,"1943","9-13; 18"

import csv,re,sys

def get_number_list(last_column):
    numbers = []
    last_column = last_column.replace(", ","; ") # ,->;
    last_column = last_column.replace(": ","; ") # :->;
    for range_str in last_column.split(';'):     # multiple numbers/sequences separated by ;
        range_str = range_str.strip()            # trim
        range_str = range_str.replace("[","")    # [num] means manually determined page number 
        range_str = range_str.replace("]","") 

        if '-' in range_str: # sequence
            m = re.findall("(\d+)\s*-+\s*(\d+)",range_str)
            if m and len(m)==1:
                m=m[0]
                if len(m)==2:
                    start = int(m[0])
                    end = int(m[1])
                    numbers.extend(range(start, end + 1))
                else:
                    print("WARNING 1",last_column)
            else:
                print("WARNING 2",last_column)
        else:
            range_str = re.sub(r" .*","",range_str) # remove everything after first space (CHECKME!)

            if range_str.isdigit():  # single number
                numbers.append(int(range_str))
            else:
                print("WARNING 3",last_column, "HUIDIGE STUK:",range_str)
    return numbers

with open('artikelen.csv', 'r') as file:
    reader = csv.reader(file)
    next(reader)  # skip header

    for row in reader:
        last_column = row[-1]
        try:
            numbers = get_number_list(last_column)
        except Exception as e:
            print(e,row);
        print(row, numbers)

copy all files from textfile to folder without hierarchy / folder structure

import os
import shutil
from tqdm import tqdm

def copy_files_to_destination(file_path, destination):
    with open(file_path, 'r') as file:
        files = list(file.readlines())
        for line in tqdm(files):
            absolute_path = line.strip()
            if os.path.isfile(absolute_path):
                file_name = os.path.basename(absolute_path)
                dest_path = os.path.join(destination, file_name)
                shutil.copy(absolute_path, dest_path)
                #print(f"Bestand gekopieerd: {absolute_path} naar {dest_path}")

source_file = 'xmls.txt'
destination_folder = 'xml/'

copy_files_to_destination(source_file, destination_folder)

Render AltoXML data to JPG for quality check

from pathlib import Path
import os,cv2
from alto import parse_file, String
import numpy as np
from PIL import Image, ImageDraw, ImageFont
from tqdm import tqdm

# BASE_FOLDER = "xxxx 2010-2022 50119/"
BASE_FOLDER = "xxxx 2010-2022 14215/"
IMAGE_FOLDER = BASE_FOLDER+"JPG"
ALTO_FOLDER = BASE_FOLDER+"Alto-XML"
OUTPUT_FOLDER = BASE_FOLDER+"output"

def list_files(folder, scheme): 
    return list(Path(folder).rglob(scheme))

def list_image_files():
    return list_files(IMAGE_FOLDER,"*.jpg")

def list_alto_files():
    return list_files(ALTO_FOLDER,"*.xml")

def get_alto_file(file_id):
    return Path(str(make_path(ALTO_FOLDER, file_id, ".xml")).replace(".xml","_alto.xml"))

def get_image_file(file_id):
    return make_path(ALTO_FOLDER, file_id, ".jpg")

def get_output_file(file_id):
    return make_path(OUTPUT_FOLDER, file_id, ".jpg")

def get_id_from_path(path):
    return Path(os.path.basename(os.path.dirname(path))).joinpath(path.stem)

def make_path(base_folder, id, suffix):
    file = Path(base_folder).joinpath(id).with_suffix(suffix)
    file_folder = file.parent
    file_folder.mkdir(parents=True, exist_ok=True)
    return file

#######################

font = ImageFont.truetype("arial.ttf", 24)
for image_file in tqdm(list_image_files()[:5]):
    img = Image.open(str(image_file))
    draw = ImageDraw.Draw(img)
    file_id = get_id_from_path(image_file)
    alto_file = get_alto_file(file_id)
    output_file = get_output_file(file_id)
    alto = parse_file(str(alto_file))
    for line in alto.extract_text_lines():
        for word in line.strings:
            if isinstance(word, String):
                (x,y) = p1 = tuple(int(num) for num in (word.hpos,word.vpos))
                (w,h) = tuple(int(num) for num in (word.width,word.height))
                p2 = (x+w,y+h)
                draw.rectangle((x, y, x+w, y+h), outline="blue")
                draw.text((x, y+h-10), word.content, font=font, fill="red")

    nieuwe_breedte = int(img.width * 0.5)
    nieuwe_hoogte = int(img.height * 0.5)
    img = img.resize((nieuwe_breedte, nieuwe_hoogte))
    img.save(str(output_file))

Groeperen met Pandas

import pandas as pd
df = pd.read_excel('data/all-rows.xlsx')

df = df[df['CODE'].isin(["1202.xx","1202.xx",........])]

groepen = df.groupby(['Straatnaam', 'Huisnummer'])
groepen = groepen.filter(lambda x: len(x) > 1)
groepen = groepen.groupby(['Straatnaam', 'Huisnummer'])

with open("tmp.tsv","w") as out:
    for naam, groep in groepen:
        print(" ".join(naam),file=out)
        for index, rij in groep.iterrows():
            rij = rij.fillna('')
            print(rij["ID"], rij["CODE"], " ", rij['Achternaam'],  rij['Voorna(a)m(en)'], rij['Geboortedatum'], rij["xxxx"], rij["Overslaan in uitvoer"], rij["Externe Identifier"], rij["Bron overlijden"], rij["GUID"], sep='\t',file=out )
        print("\n",file=out)

Download all prismic documents and images

#!/usr/bin/env python3

import os,re,requests,json,urllib,hashlib
from urllib.parse import urlparse
from collections import defaultdict

url = "https://XXXXXX.cdn.prismic.io/api/v2/documents/search?ref=XXXXXXXX"

while url:
    response = requests.get(url)
    data = response.json()
    page = data.get("page")
    results = data.get("results")
    json.dump(data, open(f"page{page}.json","w"), indent=4)

    for result in results:
        for field in result["data"]:

            if result["data"][field] and type(result["data"][field])==dict and "url" in result["data"][field]:
                url = result["data"][field]["url"]

                parsed_url = urlparse(url)
                path = parsed_url.path
                ext = os.path.splitext(path)[1]

                filename = "images/" + hashlib.md5(url.encode()).hexdigest() + ext

                print(field, filename)

                if not os.path.exists(filename):
                    urllib.request.urlretrieve(url, filename) # save 

    url = data.get("next_page")

    # break

file exists

if os.path.exists(filename):

voeg dashes toe aan GUID

def guid_dashed(s):
  return "{" + (f"{s[:8]}-{s[8:12]}-{s[12:16]}-{s[16:20]}-{s[20:]}") + "}"

splits achternaam op tussenvoegsel en achternaam

    # split achternaam / tussenvoegsel
    for rol in ["overledene","vader","moeder","partner"]:
        achternaam = item.get(f"achternaam {rol}")

        patroon = re.compile(r"^(van der|van den|van de|van het|van 't|van|der|de)", re.IGNORECASE)
        tussenvoegsel = patroon.findall(achternaam)
        achternaam = patroon.sub("", achternaam)

        if tussenvoegsel:
            item[f"tussenvoegsel {rol}"] = (" ".join(tussenvoegsel)).lower()
        item[f"achternaam {rol}"] = achternaam.strip()

re-create items based on expected keys with specific field order using list comprehension

expected_keys = ["filename", "x", "y" ] # etc
items = [{key: item[key] for key in expected_keys if key in item} for item in items]

# without list comprehension:
# new_items = []
# for item in items:
#     new_item = {}
#     for key in expected_keys:
#         if key in item:
#             new_item[key] = item[key]
#     new_items.append(new_item)

zorg dat de letter A of B altijd gevolgd wordt door een spatie wanneer het direct gevolgd wordt door een cijfer.

re.sub(r"(A|B)(\d)", r"\1 \2", input_string)

fuzzy lookup for dates in different Dutch formats

dates = get_all_dates(datetime(1800,1,1))

lut  = { get_dutch_date_with_dashes(date):date for date in dates }
lut |= { get_dutch_date_with_written_month(date):date for date in dates }
lut |= { get_dutch_date_fully_written(date):date for date in dates }

a = fuzzy_extract("5-10_1950t", lut.keys())
b = fuzzy_extract("5 oktber 1950", lut.keys())
c = fuzzy_extract("vijftn tober negentsnhonerd viftg", lut.keys())

print(lut[a[0]], a)
print(lut[b[0]], b)
print(lut[c[0]], c)

get all dates in a range as a list

with list comprehension and typed

def get_all_dates(start_date: datetime = datetime(1900, 1, 1), end_date: datetime = datetime.now()) -> List[datetime]:
    return [start_date + timedelta(days=d) for d in range((end_date - start_date).days+1)]

without:

def get_all_dates(start_date=datetime(1900, 1, 1), end_date=datetime.now()):
    dates = []
    date = start_date
    while date < end_date:
        dates.append(date)
        date += timedelta(days=1)
    return dates  

format Dutch dates

def get_dutch_date_with_dashes(date): # 05-12-1979  met voorloop nul
    return date.strftime("%d-%m-%Y")

def get_dutch_date_written_month(date): # 5 december 1979  zonder voorloop nul bij dag
    locale.setlocale(locale.LC_TIME, "nl_NL")
    return date.strftime("%-d %B %Y")

def get_dutch_date_fully_written(date): #vijftien december negentienhonderd negenzeventig
    locale.setlocale(locale.LC_TIME, "nl_NL")
    numbers_str = ["één","twee","drie","vier","vijf","zes","zeven","acht","negen","tien","elf","twaalf","dertien","veertien","vijftien","zestien","zeventien","achttien","negentien","twintig","eenentwintig","tweeëntwintig","drieëntwintig","vierentwintig","vijfentwintig","zesentwintig","zevenentwintig","achtentwintig","negenentwintig","dertig","eenendertig","tweeëndertig","drieëndertig","vierendertig","vijfendertig","zesendertig","zevenendertig","achtendertig","negenendertig","veertig","eenenveertig","tweeënveertig","drieënveertig","vierenveertig","vijfenveertig","zesenveertig","zevenenveertig","achtenveertig","negenenveertig","vijftig","eenenvijftig","tweeënvijftig","drieënvijftig","vierenvijftig","vijfenvijftig","zesenvijftig","zevenenvijftig","achtenvijftig","negenenvijftig","zestig","eenenzestig","tweeënzestig","drieënzestig","vierenzestig","vijfenzestig","zesenzestig","zevenenzestig","achtenzestig","negenenzestig","zeventig","eenenzeventig","tweeënzeventig","drieënzeventig","vierenzeventig","vijfenzeventig","zesenzeventig","zevenenzeventig","achtenzeventig","negenenzeventig","tachtig","eenentachtig","tweeëntachtig","drieëntachtig","vierentachtig","vijfentachtig","zesentachtig","zevenentachtig","achtentachtig","negenentachtig","negentig","eenennegentig","tweeënnegentig","drieënnegentig","vierennegentig","vijfennegentig","zesennegentig","zevenennegentig","achtennegentig","negenennegentig"]
    day = numbers_str[date.day-1]
    century = int(date.year/100)
    year_within_century = int(date.year%100)
    century_str = numbers_str[century-1]
    year_within_century_str = numbers_str[year_within_century-1] if year_within_century else ""
    return (f"{day} {date.strftime('%B')} {century_str}honderd {year_within_century_str}").strip()

merge two dicts with '|' operator

datums = get_alle_datums_als_tekst(datetime(1800,1,1))
datums |= get_alle_datums(datetime(1800,1,1))

filter out keys that are not in expected_keys

data = {"filename": "x", "aktenummer": "y", "overlijdensdatum": "z", "achternaam overledene": "FILTERED OUT"}
expected_keys = ["filename", "aktenummer", "overlijdensdatum"]
data = dict(filter(lambda x: x[0] in expected_keys, data.items()))

or for multiple items:

items = [ dict(filter(lambda x: x[0] in expected_keys, item.items())) for item in items ]

call a remote function running with Flask

URL=http://URL
INPUT_FILE=INPUT.txt
OUTPUT_FILE=OUTPUT.json
curl -J -X POST -F "file=@$INPUT_FILE" $URL > $OUTPUT_FILE

remote function implementation on server with Flask

#!/usr/bin/env python3
from flask import Flask,request,send_file
import json

app = Flask(__name__)

@app.route("/")
def index():
    return "ok"

@app.route('/run', methods=['POST'])
def run():
    input_file_path = '/tmp/INPUT.txt'
    output_file_path = '/tmp/OUTPUT.json'
    f = request.files['file']
    f.save(input_file_path)

    with \
        open(input_file_path, 'r', encoding="utf-8") as input_file, \
        open(output_file_path, 'w') as output_file:
            output = CONVERT_FILE_FUNCTION(input_file.read())
            json.dump(output, output_file, indent=2, ensure_ascii=False)

    result_file = open(output_file_path,'rb')
    return send_file(result_file, as_attachment=True, download_name='result.json')

if __name__ == "__main__":
    from waitress import serve
    print("URL:PORT")
    serve(app, host=URL, port=PORT)

harvest OAI-PMH and save as xml and json (using resumptionToken)

#!/usr/bin/env python3
import requests,json,xmltodict

base_url = "...........?verb=ListRecords"
url = base_url + "&metadataPrefix=oai_a2a&set=............"
page = 0

while url:
    print(url)
    response = requests.get(url)

    with open(f"data/xml/page-{page}.xml","w") as f:
        f.write(response.text)

    d = xmltodict.parse(response.text)
    json.dump(d,open(f"data/json/page-{page}.json","w"),indent=2)

    resumption_token = d["OAI-PMH"]["ListRecords"]["resumptionToken"]["#text"]

    if resumption_token:
        url = base_url + "&resumptionToken=" + resumption_token
        page += 1
    else:
        print("end?")
        break

selected python version...

/Applications/Xcode.app/Contents/Developer/usr/bin/python3.9

print SSL version

import ssl; print(ssl.OPENSSL_VERSION)

get version of installed modules with pip

pip list

update module with pip

pip install --upgrade openai

list amount of each 'Soort' across different json files

from collections import defaultdict
from pathlib import Path
import json

soort = defaultdict(int)
input_files = list(Path("data/json").rglob("*.json"))

for input_file_path in input_files:
    with open(input_file_path,"r") as file:    
        for item in json.load(file):
            soort[item.get("Soort")] += 1

soort = dict(sorted(soort.items(), key=lambda x:x[1], reverse=True))

print(json.dumps(soort,indent=2,ensure_ascii=False))

according to ChatGPT this code can written shorter as, thanks:

import json
import pandas as pd
from pathlib import Path

input_files = list(Path("data/json").rglob("*.json"))
df = pd.concat([pd.read_json(file) for file in input_files])
soort = df["Soort"].value_counts().to_dict()
print(json.dumps(soort,indent=2,ensure_ascii=False))

sort dict by value descending

soort = dict(sorted(soort.items(), key=lambda x:x[1], reverse=True))

default dict with int instead of lambda:0

soort = defaultdict(int)

RDF / turtle / jsonLD

see RDF

Download multiple pages of JSON from (omeka-s) API

page = 1
url = f"{API_URL}&page={page}"
while url:
    print(url)
    response = requests.get(url)
    json.dump(response.json(), open(f"data/page{page}.json","w"), indent=2)
    next = response.links.get("next")
    url = next["url"] if next else ""
    page += 1

get next link url from http response header

response = requests.get(url)
next = response.links.get("next")
if next:
    print(next["url"])

json dumps http request response header

import requests, json
response = requests.get(url)
print(json.dumps(dict(response.headers),indent=2))

item get

if "Achternaam" in item and item["Achternaam"]=="Bicker":

vs.

if item.get("Achternaam")=="Bicker":

remove double line breaks with optional whitespace in between

s = re.sub(r"\n\s*\n", "\n", s)

replace spaces by '_' from all keys in a dict

item = { x.replace(' ', '_'): v for x, v in item.items() }

pipx

pipx is a tool to help you install and run end-user applications written in Python. It's roughly similar to macOS's brew, JavaScript's npx, and Linux's apt.

It's closely related to pip. In fact, it uses pip, but is focused on installing and managing Python packages that can be run from the command line directly as applications. https://pypa.github.io/pipx/

tweepy

import tweepy,json
client = tweepy.Client("Bearer Token")
user_id=123123123
tweet_fields = ["attachments","author_id","context_annotations","conversation_id","created_at","entities","geo","id","in_reply_to_user_id","lang","referenced_tweets","reply_settings","source","text","withheld"]
tweets = []

# user tweets
#   similar for client.get_liked_tweets
for tweet in tweepy.Paginator(client.get_users_tweets,id=user_id, tweet_fields=tweet_fields, max_results=100).flatten(limit=1500):
    tweets.append(tweet.data)
with open('tweets.json', 'w') as f:
    json.dump(tweets, f, indent=2)

parse and format date

# from: 2023-01-24T10:32:01+01:00
# to: 2023-01-24 10:32
datetime.datetime.strptime(date_str, "%d/%b/%Y:%H:%M:%S %z").strftime("%Y-%m-%d %H:%M")

parse access.log files to csv

see: https://coderwall.com/p/snn1ag/regex-to-parse-your-default-nginx-access-logs

#!/usr/bin/env python3
import json,re,csv,datetime

result = []

for line in open('all.log').readlines():
    # if not re.findall(r'GET /\d{1,2} ', line): # filter on GET /1, GET /2 etc.
    #     continue

    r = re.match(r'(?P<ipaddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - \[(?P<dateandtime>.*)\] \"(?P<httpstatus>(GET|POST) .+ HTTP\/1\.1)\" (?P<returnstatus>\d{3} \d+) (\".*\")(?P<browserinfo>.*)\"',line)
    if r != None:
        result.append({'IP address': r.group('ipaddress'), 'Time Stamp': 
            datetime.datetime.strptime(r.group('dateandtime'), "%d/%b/%Y:%H:%M:%S %z").strftime("%Y-%m-%d %H:%M"),
            'HTTP status': r.group('httpstatus'), 'Return status': 
            r.group('returnstatus'), 'Browser Info': r.group('browserinfo')})

# print(result)
with open('output.csv', 'w', encoding='utf8') as file:
    writer = csv.DictWriter(file, result[0].keys())
    writer.writeheader()
    writer.writerows(result)

#with open('data.json', 'w') as fp:
#    json.dump(result, fp, indent=2) 

concat two lists

result = list1 + list2

json2csv (where json is an array of 'flat' objects)

#!/usr/bin/env python3

import sys,csv,json

if len(sys.argv)<3:
    sys.exit(f"Usage: {sys.argv[0]} input.json output.csv")

with open(sys.argv[1]) as json_file, open(sys.argv[2],'w') as csv_file:
    data = json.load(json_file)
    all_keys = set(key for row in data for key in row.keys())
    writer = csv.DictWriter(csv_file, all_keys)
    writer.writeheader()
    writer.writerows(data)

get all keys used in a list of dicts

all_keys = set()
for row in data:
    for key in row.keys():
        all_keys.add(key)
print(all_keys)    

how to retrieve in Python multiple json files from a REST server when the next_page attribute is supplied by the server?

import requests, json
url = "YOUR_API_URL"
while url:
    response = requests.get(url)
    data = response.json()
    page = data.get("page")
    json.dump(data, open(f"page{page}.json","w"), indent=4)
    url = data.get("next_page")

read formula from cells in Excel

from openpyxl import load_workbook
wb = load_workbook(filename = 'HUA-Sabine-Fuzzy-Matches-score-vanaf-50pct-19-april-2022.xlsx')
sheet = wb['Sheet1']
for row in range(1,10):
    status = sheet.cell(row=row, column=1).value

install package with sudo as root

this way also for example the wwwdata user can use the package. not sure if it's safe.

sudo su
sudo pip3 install mypackage

fuzzy regex

match = regex.search(f"(negentienhonderd){{e<=3}}", item["text"], regex.BESTMATCH)

point finding functions

def get_points_left(point, points):
    if points:
        return list(filter(lambda p: p[0]<point[0], points))

def get_points_below(point, points):
    if points:
        return list(filter(lambda p: p[1]>point[1], points))

def get_points_left_below(point, points):
    if points:
        return get_points_left(point, get_points_below(point, points))

def closest_point(point, points):
    if points:
        return points[distance.cdist([point], points).argmin()]

#example:
p2 = closest_point(p1, get_points_left_below(p, points))

template matching with opencv

def match(img, template):
  result = cv2.matchTemplate(img, template, cv2.TM_CCORR_NORMED)
  min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)
  lt = (x1,y1) = max_loc
  rb = (x2,y2) = (x1+tw,y1+th)
  return (lt,rb)

lt,rb = match(img, template)
cv2.rectangle(img, lt, rb, color=0, thickness=-1)

cv2.rectangle

cv2.rectangle(img, lt, rb, color=(b,g,r), thickness=10)

make a list unique

result = list(set(['a','b','a']))
# result = ['a','b']

create csv from a list of tuples

plaatsnamen = [("Utrecht",5), ("Amersfoort",2)]
writer = csv.writer(open("output.csv","w"))
writer.writerow(['plaatsnaam','aantal'])
writer.writerows(plaatsnamen)

sort dict

plaatsnamen = dict(sorted(plaatsnamen.items(), key=lambda item: item[1], reverse=True))

fuzzy match and convert a Dutch written/spelled date from the 20th century:

from datetime import datetime, timedelta
import sys
from rapidfuzz import fuzz
import rapidfuzz.process as fuzzy

def alle_datums_20e_eeuw_als_tekst():  # returns a dict {string:datetime}
    maanden = ["januari","februari","maart","april","mei","juni","juli","augustus","september","oktober","november","december"]

    getallen_1tm99 = ["één","twee","drie","vier","vijf","zes","zeven","acht","negen","tien","elf","twaalf","dertien","veertien","vijftien","zestien","zeventien","achttien","negentien","twintig","eenentwintig","tweeëntwintig","drieëntwintig","vierentwintig","vijfentwintig","zesentwintig","zevenentwintig","achtentwintig","negenentwintig","dertig","eenendertig","tweeëndertig","drieëndertig","vierendertig","vijfendertig","zesendertig","zevenendertig","achtendertig","negenendertig","veertig","eenenveertig","tweeënveertig","drieënveertig","vierenveertig","vijfenveertig","zesenveertig","zevenenveertig","achtenveertig","negenenveertig","vijftig","eenenvijftig","tweeënvijftig","drieënvijftig","vierenvijftig","vijfenvijftig","zesenvijftig","zevenenvijftig","achtenvijftig","negenenvijftig","zestig","eenenzestig","tweeënzestig","drieënzestig","vierenzestig","vijfenzestig","zesenzestig","zevenenzestig","achtenzestig","negenenzestig","zeventig","eenenzeventig","tweeënzeventig","drieënzeventig","vierenzeventig","vijfenzeventig","zesenzeventig","zevenenzeventig","achtenzeventig","negenenzeventig","tachtig","eenentachtig","tweeëntachtig","drieëntachtig","vierentachtig","vijfentachtig","zesentachtig","zevenentachtig","achtentachtig","negenentachtig","negentig","eenennegentig","tweeënnegentig","drieënnegentig","vierennegentig","vijfennegentig","zesennegentig","zevenennegentig","achtennegentig","negenennegentig"]

    current_date = datetime(1900, 1, 1)
    dagen = {}
    while current_date.year < 2000:
        dag = getallen_1tm99[current_date.day-1]
        maand = maanden[current_date.month-1]
        jaar = "negentienhonderd "+getallen_1tm99[current_date.year-1900-1]

        dagen[f"{dag} {maand} {jaar}"] = current_date
        current_date += timedelta(days=1)
    return dagen

def fuzzy_extract(input_str, compare_strs): #(result, match_pct, idx)
    return fuzzy.extractOne(input_str, compare_strs, scorer=fuzz.ratio)

datums = alle_datums_20e_eeuw_als_tekst()
#result,_,_ = fuzzy_extract("dertien januari negentienhonderd negenennegentig", datums.keys())
result,_,_ = fuzzy_extract("dertasdfien januaasdfri negeasdfntienahonderd negenenneasdfgentig", datums.keys())

print(result)
print(datums[result])

defaultdict in defaultdict (test)

all_first_names_and_occurences_dict = defaultdict(lambda: defaultdict(lambda: 0))

output_rows to excel using pandas

print("writing Excel file")
excel_writer = pd.ExcelWriter(config.EXCEL_OUTPUT_FILE, engine='xlsxwriter')
df = pd.DataFrame(output_rows)
df.to_excel(excel_writer, sheet_name="Sheet1", startrow=1, header=False, index=False)
excel_writer.sheets['Sheet1'].set_column(0,20, 25) # for cols 0 to 20 set width=25
(max_row, max_col) = df.shape
column_settings = [{'header': column} for column in df.columns]
excel_writer.sheets['Sheet1'].add_table(0, 0, max_row, max_col - 1, {'columns': column_settings})
excel_writer.close()

pandas groupby to tabs/sheets in Excel with table

df = pd.read_csv("overzicht.csv") 

writer = pd.ExcelWriter('test.xlsx', engine='xlsxwriter')

for aet, frame in df.groupby("AET"):
    frame.to_excel(writer, sheet_name=aet, startrow=1, header=False, index=False)

    column_settings = [{'header': column} for column in frame.columns]
    (max_row, max_col) = frame.shape
    writer.sheets[aet].add_table(0, 0, max_row, max_col - 1, {'columns': column_settings})

writer.close()

xlsxwriter with pandas

• https://xlsxwriter.readthedocs.io/working_with_pandas.html

format with 'repr', 'str' or 'ascii'

print(f"{state!r}")
print(f"{state!s}")
print(f"{state!a}")

pandas groupby (count)

df = pd.read_csv("overzicht.csv") # sep=',', engine='python', header=0, usecols=["tip", "sex", "time"]), index_col=["sex", "tip"], nrows=5, skiprows = [1,12], 
group_by_aet = df.groupby("AET")["CODE"].count()
print(group_by_aet)

trim items when splitting

input_string='ARP; CHU; KVP'
items = [x.strip() for x in input_string.split(';')]
# ['ARP', 'CHU', 'KVP'] instead of ['ARP', ' CHU', ' KVP']

oracle sql through jdbc

pip install cx_oracle

• and download the instant client
• open the dylibs with Terminal.app to Allow execution

import cx_Oracle
lib_dir = os.path.join(os.environ.get("HOME"), "Downloads", "instantclient_19_8")
cx_Oracle.init_oracle_client(lib_dir=lib_dir)
dsn = cx_Oracle.makedsn(IP_OR_HOSTNAME,'1521',service_name=SERVICE_NAME)
connection = cx_Oracle.connect(os.environ["USER"], os.environ["PASS"], dsn)
cur = connection.cursor()
for row in cur.execute("select * from TABLE;"):
  print(row)

via dotenv:

import cx_Oracle
from dotenv import load_dotenv
load_dotenv()    

def oracle_get_cursor():
    lib_dir = os.path.join(os.environ["ORACLE_LIB_DIR"])
    cx_Oracle.init_oracle_client(lib_dir=lib_dir)
    dsn = cx_Oracle.makedsn(os.environ["ORACLE_IP"],'1521',service_name=os.environ["ORACLE_SERVICE_NAME"])
    connection = cx_Oracle.connect(os.environ["ORACLE_USER"], os.environ["ORACLE_PASS"], dsn)
    cur = connection.cursor()
    return cur

query result as dictionary (single result):

def get_bestandsnaam(cur, id):
    #cur.execute(f"SELECT * FROM bestanden WHERE id={id}") # NOT SAFE
    cur.execute("""select * from bestanden where id = :id""", id=41590334) # MORE SAFE
    cur.rowfactory = lambda *args: dict(zip([d[0].lower() for d in cur.description], args))
    row = cur.fetchone()
    return row["bestandsnaam"]

list all valid xnummers recursively

def listdir_valid_xnummers_to_csv():
    folderroot = "/Volumes/SHARE$/FOLDER"
    xnummers = []

    for path in Path(folderroot).rglob('*.jpg'):
        r = re.findall(r"(?:X)(\d+)",path.name)
        if r:
            path_without_root = str(path).replace(folderroot,'') # let op! str(path) !
            xnummers.append({"nummer":int(r[0]), "path":path_without_root})

    xnummers = sorted(xnummers, key=lambda item:item['nummer'])

    writer = csv.DictWriter(open("all-valid-xnummers.csv","w"), fieldnames=["nummer","path"])
    writer.writeheader()
    writer.writerows(xnummers)

pathlib / os.path

import os
from pathlib import Path

for path in Path(folderroot).rglob('*.jpg'):
    folder = os.path.basename(os.path.dirname(path))
    print(folder, path.name)

try except

try:  
    1 / 0 
except Exception as e: 
    print(e)

with traceback:

import traceback
traceback.print_exc()

named tuples

from collections import namedtuple
Point = namedtuple('Point', ['x', 'y'])
#Rect = namedtuple('Rect', ['x', 'y', 'w', 'h'])
a = Point(1,2)
print(a.x)

list to dict

dict(zip(['x','y','w','h'], get_bounds(box_points)))

cast array to namedTuple

"You can do Row(*A) which using argument unpacking." source

from collections import namedtuple
input_array = [1705, 155, 106, 38]
Rect = namedtuple('Rect', ['x', 'y', 'w', 'h'])
bounds = Rect(*input_array)
print(bounds)
# output: Rect(x=1705, y=155, w=106, h=38)

ensure ascii=False

print(json.dumps(data,indent=2,ensure_ascii=False))

liquid templates

pip install liquidpy

from liquid import Liquid
tpl = Liquid("templates/template.xml", liquid_from_file=True) 
data = {
  "ID": item["index"],
  "ParentID": item["parentIndex"],
  "Text": item["text"]
}
result = tpl.render(**data)
print(result)

custom filters in liquidpy 'wild' mode

{% addfilter md5 %}
import hashlib
def md5(s):
    return hashlib.md5(s.encode()).hexdigest()
{% endaddfilter %}

parse macOCR result

def read_ocr(filename): # txt macOCR format
    for raw_line in open(filename).readlines():
        line_tuple = make_tuple(raw_line)
        yield({
            "box": [tuple([int(coordinate) for coordinate in coordinates]) for coordinates in line_tuple[0]],
            "text": line_tuple[1],
            "confidence": line_tuple[2]
        })

ocr_items = list(read_ocr(INPUT_MACOCR))

ignore 'illegal' unicode chars in string (?)

str(data["Text"].encode("ascii", "ignore"))

sort by integer without itemgetter but wit lambda function

relaties = sorted(relaties, key=lambda item:int(item['aantal']), reverse=True)

strip trailing numbers comma and whitespace

s = s.strip().lower().rstrip(" 1234567890,")

download and save image

urllib.request.urlretrieve(URL, local_filename)

markup html with BeautifulSoup

https://beautiful-soup-4.readthedocs.io/en/latest/

from bs4 import BeautifulSoup
soup = BeautifulSoup(html, 'html.parser')
print(soup.prettify())

get all links:

for link in soup.find_all('a'):
    print(link.get('href'))

get text of div with class:

copyright = soup.find("div", class_="copyright-authoryear").get_text().strip()

which python uses sublime?

#!/usr/bin/env python3

import sys
print(sys.executable)
sys.exit()

in my case:

/usr/local/opt/python@3.10/bin/python3.10

use pip for specific python version

/usr/local/opt/python@3.10/bin/python3.10 -m pip install bs4

requests

import requests, json

response = requests.get(URL)
# response = requests.get(URL, headers={"Accept":"application/ld+json"})

data = response.json()
print(json.dumps(data,indent=2))

on ubuntu when pip python packages end up in ~/.local/lib/python3.6/site-packages/

change user to root before pip install

sudo su

list used packages in Python script with versions

import pkg_resources
for i in pkg_resources.working_set:
  print(i.key + "==" + i.version)

list all files recursive

#!/usr/bin/env python3

from pathlib import Path

folder = "/Volumes/archiefbestanden$/Kadaster/DVD's/"

for path in Path(folder).rglob('*'):
    print(path)

join list to string ignore blanks

' '.join(filter(None, strings))

set label for tqdm progress bar

pbar = tqdm(straten) # list

for i,straat in enumerate(pbar):
  straatnaam = straat["straatnaam"]
  pbar.set_description(f"{i} {straatnaam}")

geo coordinaat opvragen van (het middelpunt van) een straat via BAG openbare_ruimte_ID op wikidata.

query="""SELECT ?straat ?straatLabel ?punt ?woonplaats ?woonplaatsLabel WHERE {
?straat wdt:P625 ?punt .
?straat wdt:P131 ?woonplaats . 
?straat wdtn:P5207  <http://bag.basisregistraties.overheid.nl/bag/id/openbare-ruimte/"""+straat_id+"""> .
SERVICE wikibase:label { bd:serviceParam wikibase:language "nl". } }"""

url = "https://query.wikidata.org/sparql"
response = requests.get(url, params={'query' : query}, headers={'Accept' : 'application/sparql-results+json'})
data = response.json()

read csv as dict etc

def csv2dict(filename):
    return csv.DictReader(open(csv_folder+filename, encoding="utf-8-sig"), delimiter=";")

def csv2lut(filename):
    return { row["Unieke waarde (niet aanpassen)"]:row["Correctie (hier corrigeren)"] 
        for row in csv2dict(filename) }

split geo point

def split_geo_point(s):
    parts = s.lower().split(" ")
    lat = parts[1].replace(")","")
    lon = parts[0].replace("point(","")
    return (lat,lon)

urlencode

print(urllib.parse.quote_plus(query))

write stderr to file

sys.stderr = open("stderr.log","w")

total seconds from time

# must be a better way...
dt = datetime.strptime(film["STARTTIJD"][:8],"%H:%M:%S").time()
startOffset = dt.hour * 3600 + dt.minute * 60 + dt.second

namedtuple

Point = namedtuple('Point', 'x y')
center = Point(10,10) # center is a namedtuple, that can be accessed either using x and y or an index (0,1)
print(center.x)

parse points and get bounds

def get_points(points_string):
    return [tuple(map(int,coords.split(','))) for coords in points_string.split(" ")] 

def get_bounds(points):
    x = min(points)[0], max(points)[0]
    y = min(points)[1], max(points)[1]
    return ((x[0],y[0],x[1],y[1]))

def is_overlapping_1D(line1, line2): # line: (xmin, xmax)
    return line1[0] <= line2[1] and line2[0] <= line1[1] # box (xmin, ymin, xmax, ymax)

def is_overlapping_2d(box1, box2):
    return is_overlapping_1D([box1[0],box1[2]],[box2[0],box2[2]]) and is_overlapping_1D([box1[1],box1[3]],[box2[1],box2[3]])

points_string = "93,1349 93,1502 162,1502 162,1349"
print(get_points(points_string))

get duration of films in folder

#!/usr/bin/env python3
import json,csv,re,glob,os,tqdm
import datetime
from time import strftime
from time import gmtime
from os.path import exists
from pathlib import Path
from tqdm import tqdm

def get_duration_ffprobe(filename):
    import subprocess, json
    result = subprocess.check_output(
        f'ffprobe -v quiet -show_streams -select_streams v:0 -of json "{filename}"',
        shell=True).decode()
    fields = json.loads(result)['streams'][0]
    return fields['duration']

# GET DURATION OF ALL FILMS
movie_folder = "MOVIE_FOLDER/"

files = glob.glob(movie_folder+"*.mp4", recursive=False)

writer = csv.DictWriter(open("film-duration.csv","w"), fieldnames=["name","duration"]) #, delimiter=',', quoting=csv.QUOTE_ALL, dialect='excel')
writer.writeheader()

for filename in tqdm(files):
    row = {}
    row["name"] = os.path.basename(filename)
    row["duration"] = strftime("%H:%M:%S:00", gmtime(float(get_duration_ffprobe(filename))))
    writer.writerow(row)

difflib

https://docs.python.org/3/library/difflib.html (tip van Lars)

fuzzy matching in strings

https://gist.github.com/companje/93f6061629ac27a2027a77888effd6ad

yield

The yield statement suspends function’s execution and sends a value back to the caller, but retains enough state to enable function to resume where it is left off. When resumed, the function continues execution immediately after the last yield run. This allows its code to produce a series of values over time, rather than computing them at once and sending them back like a list. https://www.geeksforgeeks.org/use-yield-keyword-instead-return-keyword-python/

def x():
    for i in [5,4,2]:
        yield(i)

for i in x():
    print(i)

uuid / guid

import uuid
print(uuid.uuid4().hex)

string padding

mystring.rjust(10, '0')) 

any

contains_thans = any(thans_spelling in raw_country_str for thans_spelling in ["thans", "th.", "th "])

usage

import sys,os
from sys import argv

if len(argv)!=3:
  sys.exit("Usage: "+os.path.basename(argv[0])+" {INPUT_CSV} {OUTPUT_FILE}")
input_filename = argv[1]
output_filename = argv[2]

unieke waarden per kolom in csv

• zie gist: https://gist.github.com/companje/72ddf8f4ddba271580af2a55f62bcfad

parallel processing

from joblib import Parallel, delayed

def DoSomething(filename, param2, param2):
  #...
  return result

results = Parallel(n_jobs=8)(
  delayed(DoSomething)(filename, param2, param3)
  for filename in tqdm(filenames)
)

get number of processor cores

import psutil as psutil
print(psutil.cpu_count(logical=True))

progress bar

from tqdm import tqdm
#...
for filename in tqdm(filenames)
  #...

request

from urllib.request import urlopen
import urllib.parse
#...
request = urlopen(url)
data = json.load(request)
json.dump(data, open(cache_filename,"w"), indent=4)

defaultdict with numbers

aets = defaultdict(lambda: 0)
for ....
  aets[row["CODE"]] += 1

serial

#!/usr/bin/env python3

# on Sanyo: type file.asm > aux

import serial

ser = serial.Serial('/dev/tty.usbmodem1301',1200)

while True:
    x = ser.read()
    print(x.decode('ascii'), end="")

ser.close()

parse and format date

import datetime
try:
  isodate = datetime.datetime.strptime(datum, '%d-%m-%Y').strftime('%Y-%m-%d')
except ValueError:
  pass # just skip invalid/incomplete dates

maak spreadsheet met velden als kolommen per eenheid

#!/usr/bin/env python3
import csv,re
from collections import defaultdict

filename = "alle-personen.csv"
output_filename = "output.csv"
fixed = ["ID","GUID","CODE","BESTANDSNAAM"]
flex_key = "PROMPT"
flex_value = "WAARDE"

header = fixed.copy()
items = defaultdict(dict)

for row in csv.DictReader(open(filename)):

    row["WAARDE"] = row["WAARDE"].replace("\n"," ") # replace line breaks by spaces

    # create or get item
    item = items[row["ID"]]

    # add fixed fields
    for k,v in row.items():
        if k in fixed:
            item[k] = v

    # add flex fields
    item[row[flex_key]] = row[flex_value]

    # update header
    header.append(row[flex_key]) if row[flex_key] not in header else None

# output to csv
writer = csv.DictWriter(open(output_filename,"w"), fieldnames=header) #, delimiter=',', quoting=csv.QUOTE_ALL, dialect='excel')
writer.writeheader()
writer.writerows(items.values())

replace broken words based on lookup table

#!/usr/bin/env python3

import re,csv
from collections import defaultdict
import os.path

reader = csv.DictReader(open("Gekke tekens.csv"), delimiter=',')
lijst = [dict(d) for d in reader]

for line in open("vreemde_tekens2-gemaakt-via-grep-commando.txt").readlines():

    filepath = "../test.documentatie.org-met-lfs/test.documentatie.org/data/wp/"+line.split(':')[0].strip()
    basename = os.path.basename(filepath)

    if not os.path.isfile(filepath):
        print("NOT FOUND",filepath)
        continue

    with open(filepath) as infile:
        data = infile.read()

        for li in lijst:
            data = data.replace(li["fout"],li["gecorrigeerd"])        

        with open("tmp/"+basename,'w',encoding="utf-8") as outfile:
            outfile.write(data)

find broken characters with context

#!/usr/bin/env python3

import re,csv
from collections import defaultdict
from operator import itemgetter

results = defaultdict(list)

for line in open("vreemde_tekens2-gemaakt-via-grep-commando.txt").readlines():

    filepath = line.split(':')[0].strip()
    data = "".join(line.split(':')[1:])

    matches = re.findall(r'([a-zA-Z�]+)(�)([a-zA-Z�]+)',data)

    for m in matches:
        s = "".join(m)
        results[s].append("http://test.documentatie.org/data/wp/"+filepath)

writer = csv.writer(open("result.csv","w"))

for k,v in results.items():
    k2 = re.sub(r"�","",k)
    writer.writerow([k,k2,v[0],len(v)])

find hexstring in files recursively

#!/usr/bin/env python3

import glob, re

files = glob.glob('../test.documentatie.org-met-lfs/test.documentatie.org/data/wp/**/*.htm*', recursive=True)

for filename in files:

    with open(filename, "rb") as f:

        f1 = re.search(b'\xEF\xBF\xBD', f.read())

        if f1:
            print(filename,"\t",f1)

combine 2 cvs (database tables) to 1 json file with hierarchy

#!/usr/bin/env python3

import csv, json

uitvoergegevens = { row["ID"]:row for row in csv.DictReader(open("uitvoergegevens.csv", encoding="cp1252")) }
uitvoervelden = { row["ID"]:row for row in csv.DictReader(open("uitvoervelden.csv", encoding="cp1252")) }

for row in uitvoervelden.values():

    ugn = uitvoergegevens[row["UGN_ID"]]

    if "uitvoervelden" not in ugn:
        ugn["uitvoervelden"] = []
    else:
        ugn["uitvoervelden"].append(row)

json.dump(uitvoergegevens, open("result.json", "w"), indent=2)

read CSV file as dictionary with primary key

uitvoergegevens = { row["ID"]:row for row in csv.DictReader(open("uitvoergegevens.csv", encoding="cp1252")) }
uitvoervelden = { row["ID"]:row for row in csv.DictReader(open("uitvoervelden.csv", encoding="cp1252")) }

csv2xlsx - csv to excel

#!/usr/bin/env python3
# source: https://stackoverflow.com/questions/17684610/python-convert-csv-to-xlsx

import os
import glob
import csv
from xlsxwriter.workbook import Workbook

for csvfile in glob.glob(os.path.join('.', '*.csv')):
    workbook = Workbook(csvfile[:-4] + '.xlsx')
    worksheet = workbook.add_worksheet()
    with open(csvfile, 'rt', encoding='utf8') as f:
        reader = csv.reader(f)
        for r, row in enumerate(reader):
            for c, col in enumerate(row):
                worksheet.write(r, c, col)
    workbook.close()

python csv 'list' reader

file = open("file.csv")
reader = csv.reader(file)

inline print for each result

[ print(result) for result in results ]

get key and value for the items in a dict

for k,v in scores.items():
  print(k,v)

get the object with the max value in a dict

model = max(scores, key = scores.get) 
# equivalent to: 
model = max(scores, key = lambda k : scores.get(k))  

get filename from path

file_name = os.path.basename(file_path)

recursive filelist by type

file_paths = glob.glob(folder + '/**/*.jpg', recursive=True)

semi transparent rectangle opencv python

def rechthoek(img, leftTop, rightBottom, color=(255,255,0), opacity=.2):
    (x1, y1) = leftTop
    (x2, y2) = rightBottom
    sub_img = img[y1:y2, x1:x2]
    white_rect = np.ones(sub_img.shape, dtype=np.uint8)
    white_rect[:,:] = color
    res = cv2.addWeighted(sub_img, 1-opacity, white_rect, opacity, 1.0)
    img[y1:y2, x1:x2] = res

problem with scipy dependency when installing easyocr through pip

it helped to install scipy using brew instead of pip

brew install scipy
pip install easyocr

which python?

ls -l /usr/local/bin/python3
ls -l /usr/local/bin/python

create image tiles / sprite sheet with python, opencv and numpy

import cv2
import numpy as np

margin=5
cols=5
rows=10

stamp = cv2.imread("50p.jpeg")
stamp_h,stamp_w = stamp.shape[:2]

height=stamp_h*rows + (margin*rows+1)
width=stamp_w*cols + (margin*cols+1)

img = np.ones((height,width,3), np.uint8) * 255 # white

for row in range(0,rows):
  for col in range(0,cols):
    print(row,col)

    x=col*stamp_w + (col+1)*margin
    y=row*stamp_h + (row+1)*margin

    img[y:y+stamp_h,x:x+stamp_w] = stamp

cv2.imwrite("sheet.jpg", img)

cv2.namedWindow("test",cv2.WINDOW_NORMAL)
cv2.setWindowProperty("test",cv2.WND_PROP_FULLSCREEN,cv2.WINDOW_FULLSCREEN)
cv2.setWindowProperty("test",cv2.WND_PROP_FULLSCREEN,cv2.WINDOW_NORMAL)
cv2.imshow("test", img)
cv2.waitKey()

create an empty image

import cv
import numpy as np

height=1000
width=1000

img = np.zeros((height,width,3), np.uint8)

img[:,0:width//2] = (255,0,0)      # (B, G, R)
img[:,width//2:width] = (0,255,0)

cv2.imshow("test", img)
cv2.waitKey()

info about current python and system settings

import sys
print(sys.version)
print(sys.executable)
print(sys.path)

enumerate

for img_index, img_file_name in enumerate(img_file_names):

dictreader

f = open(f"data/file.csv", "r", encoding="utf-8-sig")
reader = csv.DictReader(f, delimiter=',')
data = [dict(d) for d in reader]
f.close()

defaultdict

from collections import defaultdict
my_dict = defaultdict(lambda: 0)   # maakt een dict met default waarde 0 aan voor items
other_dict = defaultdict(list)     # maakt een dict met default waarde een list []

tuple

from ast import literal_eval as make_tuple
(coords, text, conf) = make_tuple("[([[22, 15], [373, 15], [373, 89], [22, 89]], 'Aaldering =', 0.46745234890467213)]")

tuples!

https://www.studytonight.com/python/tuples-in-python

unpack values from csv row into variables

#!/usr/bin/env python3

import json,csv,sys,glob
from sys import argv

with open("TESTSERVER-aktenummers-bestandsnamen.csv") as f:
  reader = csv.DictReader(f)   #"CODE","ID","WAARDE","BESTANDSNAAM"
  for row in reader:
    (code, id, aktenummer, bestandsnaam) = row.values()
    print(code,id,aktenummer,bestandsnaam)

join

' '.join(filter(None,[item["fname"],item["prefix"],item["sname"]])) + # filter(None,..) is to get rid of double spaces

dictreader with custom delimiter and fieldnames

reader = csv.DictReader(f, delimiter=";", fieldnames=["fname","prefix","sname","bdate","publish","file_id","remark"])

filter

filteredList = filter(lambda item: item["publish"] == "ja", list(reader))

sort with 'itemgetter'

from operator import itemgetter
#...
sortedList = sorted(filteredList, key=itemgetter('sname')) 

map

lines = map(lambda item: item["name"], sortedList)

utf-8 with BOM

print(u'\ufeff',end='') # write UTF8 BOM signature without linebreak

recursive findall using xPath .//

for textline in xml.findall('.//TextLine'):  
  #...

BoundingBox

found here: https://techoverflow.net/2017/02/23/computing-bounding-box-for-a-list-of-coordinates-in-python/ and fixed a bug.

class BoundingBox(object):
    def __init__(self, points):
        self.minx, self.miny = float("inf"), float("inf")
        self.maxx, self.maxy = float("-inf"), float("-inf")
        for x, y in points:
            self.minx = min(x,self.minx)
            self.maxx = max(x,self.maxx)
            self.miny = min(y,self.miny)
            self.maxy = max(y,self.maxy)
    @property
    def width(self):
        return self.maxx - self.minx
    @property
    def height(self):
        return self.maxy - self.miny
    def __repr__(self):
        return "BoundingBox(minX={}, minY={}, maxX={}, maxY={})".format(
            self.minx, self.miny, self.maxx, self.maxy)

DictWriter

writer = csv.DictWriter(sys.stdout, fieldnames=["id","image", "etc..."], delimiter=',', quoting=csv.QUOTE_ALL, dialect='excel')
writer.writeheader()
#...
writer.writerow(item)

split / map / list comprehension

# input: 2737,1248 2787,1256 ...

# split by space and comma
coords = [coord.split(",") for coord in coords.split(" ")]
# output: [['2737', '1248'], ['2787', '1256'],...]

# cast to int
coords = [(int(float(a)), int(float(b))) for a,b in coords]
# output: [(2737, 1248), (2787, 1256), ...]

PageXML to CSV

• https://github.com/hetutrechtsarchief/pagexml2csv

read from multiple files supplied as arguments

for example ./script.py *.JSON

#!/usr/bin/env python3
import json,csv,sys,glob
from sys import argv

for filename in argv[1:]:
  with open(filename) as json_file:
    data = json.load(json_file)
    #...

get text after last /

str.rpartition("/")[-1]

from CSV to JSON without pandas

#!/usr/bin/env python 

import csv
import json

data = []

with open("INPUT.csv") as f:
    reader = csv.DictReader(f, delimiter=";")

    for row in reader:
        data.append(row)

with open("OUTPUT.json", "w") as f:
    json.dump(data, f, indent=4)

enumerate for loop

for i, col in enumerate(header):
   print(i,col)

regex

import re
s = re.sub(r"[\"\n\r\\]", "", s) # strip double quotes "
s = re.sub(r"<[^>]*>", "", s); # strip pseudo html tags

join

print("; ".join(item) + " .")

Python 3 as default on Mac

See: https://opensource.com/article/19/5/python-3-default-mac

alias python=/usr/local/bin/python3
alias pip=/usr/local/bin/pip3

Guide to Python function decorators=

http://thecodeship.com/patterns/guide-to-python-function-decorators/

4 interessante stukjes Python

def logUser(name, age, length):
    print('name: ', name)
    print('age: ', age)
    print('length: ', length)

#logUser('alex', 29, 1.75)
user = ('alex', 29, 1.75)
logUser(*user)

def logUser(name, age, length):
    print('name: ', name)
    print('age: ', age)
    print('length: ', length)

# logUser(age=28, length=1.75, name='alex')
user = {'age':28, 'length':1.75, 'name':'alex'}
logUser(**user)

def log(*args):
    print(args)
    message = ""
    for item in args:
        message += item+' ';
    print(message)

log('hallo', '2134', 'fsfdsf')

def logUser(**kwargs):
    print('name: ', kwargs['name'])
    print('age: ', kwargs['age'])
    print('length: ', kwargs['length'])

logUser(age=28, length=1.75, name='alex')

SimpleHTTPServer

python -m SimpleHTTPServer 8000

kivy

http://kivy.org/#home

ImportError: No module named NK.gui.app

In NinjaKittens folder:

export PYTHONPATH=.

gui

• http://kivy.org

libxml / libxml2 / lxml

untested:

brew install libxml2

or

sudo port install py25-lxml

install easy_install

http://pypi.python.org/pypi/setuptools#downloads

sh setuptools-0.6c9-py2.4.egg --prefix=~

Mogelijke oplossing voor problemen met Python op OSX Lion

• http://www.niconomicon.net/blog/2011/10/09/2120.wrestling.python.snow.leopard
• http://stackoverflow.com/questions/6886578/how-to-install-pycairo-1-10-on-mac-osx-with-default-python

version

python --version

location of python

type python

or

which python

info about executable

file /usr/local/bin/python

set python path (for macports?)

see also: [[macports]]

export PYTHONPATH=/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages