[Python] Add ibook creation date from sqlite Books database in macos

I tried to edit this code in Python found in https://github.com/shrsv/ibooks_highlights_export in order to have the creation date for the book in the final markdown file, which should look like this:

---
asset_id: 3AB4BC5DE2CAAEE41EBF569B5CD1B70F
author: REISNER MARC
modified_date: '2017-09-02T14:18:36'
creation_date_book: '2015-04-03T11:10:04'
title: Cadillac Desert
---


The original doesn't have the creation_date_book attribute, it just displays the creation date for the annotation.

The tentative solution is the following:

import datetime as dt
import pathlib
import re
import yaml
from typing import Any, Dict, List, Optional

import frontmatter
from dateutil import parser as duparser
from slugify import slugify

from ibooks_highlights.ibooksdb import SqliteQueryType
from ibooks_highlights.util import (NS_TIME_INTERVAL_SINCE_1970,
TEMPLATE_ENVIRONMENT, cmp_to_key,
query_compare_no_asset_id)


class BookMetadataError(Exception):
pass


class Annotation:
def __init__(
self,
location: str,
selected_text: str = None,
note: str = None,
represent_text: str = None,
chapter: str = None,
style: str = None,
modified_date: dt.datetime = None,
creation_date: dt.datetime = None,
is_deleted: bool = False,
) -> None:

# require non-deleted annotations to have selected text or be a note
if not is_deleted and (selected_text is None) and (note is None):
raise ValueError("specify either selected_text or note")

self.location = location
self.selected_text = selected_text

if represent_text is not None:
represent_text = represent_text.strip()
self.represent_text = represent_text

self.chapter = chapter
self.style = style
self.note = note
self.modified_date = modified_date
self.creation_date = creation_date
self.is_deleted = is_deleted

def __repr__(self) -> str:
return (
f"<Annotation:{self.selected_text[:70]}, "
f"Created:{self.creation_date}, Modified:{self.modified_date}>"
)

def __getitem__(self, key: str) -> Any:
return getattr(self, key)


class Book:
def __init__(
self,
#creation_date_book: dt.datetime = None,
asset_id: str = None,
filename: pathlib.Path = None,
root: pathlib.Path = None,
) -> None:

args_present = asset_id is not None
file_present = filename is not None

if args_present == file_present:
raise ValueError("specify either asset_id or filename")
#self.creation_date: Optional[dt.datetime] = None
#self._modified_date: Optional[dt.datetime] = None
self._annotations: List[Annotation] = []
self._sync_notes = True

if args_present:
self._asset_id = asset_id
self._author: Optional[str] = None
self._title: Optional[str] = None
self.creation_date_book: Optional[dt.datetime] = None
self._prev_content: Optional[str] = None
self._reader_notes = ""

if file_present:
self._process_file(filename, root)

def __repr__(self) -> str:
return f"<Book:{self.asset_id} {self.title}>"

def _process_file(self, filename: pathlib.Path, root: pathlib.Path) -> None:

if root is not None:
self._filename = filename.relative_to(root)
else:
self._filename = filename.name

try:
book = frontmatter.load(filename)
except yaml.YAMLError:
raise BookMetadataError("no frontmatter present")

if "asset_id" not in book.keys():
raise BookMetadataError("asset_id missing")

self._asset_id = book["asset_id"]
self._author = book["author"]
self._title = book["title"]
self.creation_date_book = duparser.parse(book["creation_date_book"])



if "sync_notes" in book.keys():
self._sync_notes = bool(book["sync_notes"])

self._prev_content = book.content

self._reader_notes = ""

reader_notes_start = None
ibooks_notes_start = None

prev_content_lines = self._prev_content.splitlines()

for i, line in enumerate(prev_content_lines):
if """<a name="my_notes_dont_delete"></a>""" in line:
reader_notes_start = i
if """<a name="ibooks_notes_dont_delete"></a>""" in line:
ibooks_notes_start = i

# if not present, abort
if reader_notes_start is None and ibooks_notes_start is None:
return

# if same line, that's not good
if reader_notes_start == ibooks_notes_start:
raise BookMetadataError("Note section identifiers on same line")

# if different line, select the appropriate portion of the content
if reader_notes_start is None:
reader_lines = prev_content_lines[3:ibooks_notes_start]
elif reader_notes_start < ibooks_notes_start:
reader_lines = prev_content_lines[
reader_notes_start + 1 : ibooks_notes_start
]
else:
reader_lines = prev_content_lines[reader_notes_start + 1 :]

self._reader_notes = "\n".join(reader_lines).strip()

def __str__(self) -> str:
asset_id = self._asset_id[:8].ljust(8)
mod = " "
if self.is_modified:
mod = "*"
return f"{asset_id} {mod} {self.num_annotations}\t{self._title}"

def _yaml_str(cls, txt: str) -> str:
exp = "[^A-Za-z0-9 ]+"
return re.sub(exp, "", txt)

@property
def author(self) -> str:
return self._author

@author.setter
def author(self, value: str) -> None:
if value is None:
value = "Unknown"
self._author = self._yaml_str(value)

@property
def title(self) -> str:
return self._title

@title.setter
def title(self, value: str) -> None:

if value is None:
value = "Unknown"
self._title = self._yaml_str(value)

slug = slugify(value)
asset_id = self._asset_id[:8].lower()
self._filename = f"{slug}-{asset_id}.md"

@property
def asset_id(self) -> str:
return self._asset_id

@property
def is_modified(self) -> bool:
if self._modified_date is None:
if self.num_annotations > 0:
return True
else:
return False

if len(self._annotations) == 0:
return False

anno_max = max([anno.modified_date for anno in self._annotations])
return anno_max > self._modified_date

@property
def annotations(self) -> List[Annotation]:
return self._annotations

@annotations.setter
def annotations(self, anno: List[Annotation]) -> None:
self._annotations = anno
self._annotations.sort(key=cmp_to_key(query_compare_no_asset_id))

@property
def num_annotations(self) -> int:
return len(self._annotations)

@property
def prev_content(self) -> str:
return self._prev_content

@property
def content(self) -> str:
template = TEMPLATE_ENVIRONMENT.get_template("markdown_template.md")

# print(self._reader_notes[:1000])

# filter deleted annotations
annotations = [anno for anno in self.annotations if not anno.is_deleted]

md = template.render(
title=self._title,
author=self._author,
creation_date_book=self.creation_date_book,
highlights=annotations,
reader_notes=self._reader_notes,
)
return md

def write(self, path: pathlib.Path) -> None:

if not path.is_dir():
raise NotADirectoryError(f"{str(path)} is not a directory")

if not self._sync_notes:
print("sync locked for", self._title)
return

print("updating", self._title)

mod_date = max([anno.modified_date for anno in self._annotations])
mod_date_str = mod_date.isoformat()
creat_date = max([anno.creation_date for anno in self._annotations])
creat_date_str = creat_date.isoformat()

creation_date_book_iso=creation_date_book.isoformat()

fmpost = frontmatter.Post(
self.content,
asset_id=self._asset_id,
title=self.title,
author=self.author,
modified_date=mod_date_str,
creation_date=creat_date_str,
creation_date_book=creation_date_book_iso,
)

fn = path / self._filename

# make necessary subdirs
fn.parent.mkdir(parents=True, exist_ok=True)

with open(fn, "w") as f:
s = frontmatter.dumps(fmpost)
f.write(s)


class BookList:

path: pathlib.Path
books: Dict[str, Book] = {}

def __init__(self, path: pathlib.Path) -> None:

if not path.is_dir():
raise NotADirectoryError(f"{str(path)} is not a directory")

self.path = path

if self.path.exists():
self.books = self._load_books(self.path)

def __repr__(self) -> str:
return f"<BookList:{str(self.path)}>"

def _load_books(self, path: pathlib.Path) -> Dict[str, Book]:

book_files = path.glob("**/*.md")

md_books = {}
for bf in book_files:
try:
book = Book(filename=bf, root=path)
md_books[book.asset_id] = book
except BookMetadataError:
pass

return md_books

def _get_create_book(self, asset_id: str) -> Book:

if asset_id not in self.books:
book = Book(asset_id=asset_id)
self.books[asset_id] = book

return self.books[asset_id]

def populate_annotations(self, annos: SqliteQueryType) -> None:

res = [
r
for r in annos
if r["asset_id"] is not None
and ((r["selected_text"] is not None) or (r["note"] is not None))
]

for r in res:
book = self._get_create_book(str(r["asset_id"]))
if book.title is None:
book.title = str(r["title"])
if book.author is None:
book.author = str(r["author"])

if book.creation_date_book is None:
book.creation_date_book=dt.datetime.fromtimestamp(
NS_TIME_INTERVAL_SINCE_1970 + int(r["creation_date_book"])
),



anno_group: Dict[str, List[Annotation]] = {}
for r in res:
asset_id = str(r["asset_id"])
if asset_id not in anno_group:
anno_group[asset_id] = []

location = str(r["location"]) if r["location"] else None
selected_text = str(r["selected_text"]) if r["selected_text"] else None
note = str(r["note"]) if r["note"] else None
represent_text = str(r["represent_text"]) if r["represent_text"] else None
chapter = str(r["chapter"]) if r["chapter"] else None
style = str(r["style"]) if r["style"] else None
is_deleted = bool(r["is_deleted"])

anno = Annotation(
location=location,
selected_text=selected_text,
note=note,
represent_text=represent_text,
chapter=chapter,
style=style,
modified_date=dt.datetime.fromtimestamp(
NS_TIME_INTERVAL_SINCE_1970 + int(r["modified_date"])
),
creation_date=dt.datetime.fromtimestamp(
NS_TIME_INTERVAL_SINCE_1970 + int(r["creation_date"])
),
is_deleted=is_deleted,
)
anno_group[asset_id].append(anno)

for asset_id, anno_itr in anno_group.items():
self.books[asset_id].annotations = anno_itr

def write_modified(self, path: pathlib.Path = None, force: bool = False) -> None:

if path is None:
path = self.path

if not path.is_dir():
raise NotADirectoryError(f"{str(path)} is not a directory")

path.mkdir(parents=True, exist_ok=True)

for book in self.books.values():
if (not book.is_modified) and (not force):
continue
book.write(path)




the ibooksdb.py file has been modified in order to get the creation date for the book adding the following:

books.ZBKLIBRARYASSET.ZCREATIONDATE as creation_date_book


However I get these errors:

TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType'


or

KeyError: 'creation_date_book'


What am I doing wrong? Any help is kindly appreciated.

Continue reading...