pipless/pipgen.py

#!/usr/bin/env python3
# PyPI ebuild autogenerator, written by ~keith
import requests
import mock
import setuptools
import subprocess
import os
import shutil
import traceback
import importlib.util
import pprint
import re
import argparse

def get_versions(name: str) -> list:
	resp = requests.get(f'https://pypi.org/pypi/{name}/json')
	json = resp.json()
	return sorted(json['releases'].keys())

def get_setuptools_deps(exec_dir: str) -> list:
	old_dir = os.getcwd()
	try:
		with mock.patch.object(setuptools, 'setup') as mock_setup:
			spec = importlib.util.spec_from_file_location('setup', exec_dir.rstrip('/') + '/setup.py')
			module = importlib.util.module_from_spec(spec)
			os.chdir(exec_dir)
			spec.loader.exec_module(module)
			_, kwargs = mock_setup.call_args
			return kwargs.get('install_requires', [])
	finally:
		os.chdir(old_dir)

def pkg_exists(gentoo_name: str) -> bool:
	return os.path.exists('/var/db/repos/gentoo/' + gentoo_name) or os.path.exists('/var/db/repos/pipless/' + gentoo_name)

RE_PIP_DEP = re.compile(r'^([a-zA-Z0-9._-]+)\s*(\[[a-zA-Z0-9._,\s-]+\])?\s*(?:(<=?|!=|===?|>=?|~=)\s*([a-zA-Z0-9._*+!-]+))?(\s*,.*)?')
def translate_pip_dep(pip_dep: str) -> str:
	match = RE_PIP_DEP.match(pip_dep)
	if not match:
		print(f'WARN: cannot parse pip_dep: {pip_dep}')
		return 'UNTRANSLATABLE: ' + pip_dep

	name = match.group(1)
	py_useflags = match.group(2)
	version_cmp = match.group(3)
	version = match.group(4)
	comma = match.group(5)

	if ';' in pip_dep:
		print(f'WARN: ignoring environment markers for {name}: {pip_dep.partition(";")[2]}')
	if comma:
		print(f'WARN: ignoring extra constraints for {name}: {comma.partition(";")[0]}')
	if py_useflags:
		print(f'WARN: ignoring py_useflags for {name}: {py_useflags}')

	gentoo_name = 'dev-python/' + name.replace('_', '-').replace('.', '-')
	if not pkg_exists(gentoo_name):
		if pkg_exists(gentoo_name.lower()):
			gentoo_name = gentoo_name.lower()
		elif pkg_exists('dev-python/' + name):
			gentoo_name = 'dev-python/' + name
		elif pkg_exists('dev-python/' + name.lower()):
			gentoo_name = 'dev-python/' + name.lower()
		else:
			print(f'WARN: gentoo package not found for {name}, defaulting to {gentoo_name}')

	if not version_cmp:
		return gentoo_name
	elif version_cmp in ('==', '==='):
		return f'={gentoo_name}-{version}'
	elif version_cmp == '~=':
		return f'>={gentoo_name}-{version}'
	else:
		return f'{version_cmp}{gentoo_name}-{version}'

def get_package(name: str) -> dict:
	resp = requests.get(f'https://pypi.org/pypi/{name}/json')
	json = resp.json()

	git_url = json['info']['project_urls'].get('Source Code', json['info']['home_page']).rstrip('/') + '.git'

	pkg_data = {
		'pypi_name': json['info']['name'],
		'version': json['info']['version'],
		'description': json['info']['summary'],
		'pypi_url': json['info']['package_url'],
		'home_url': json['info']['home_page'],
		'license': json['info']['license'],
		'git_url': git_url,
		'setuptools_deps': [],
		'dependencies': [],
	}

	print(f"Got {pkg_data['pypi_name']} version {pkg_data['version']}")

	try:
		print("Attempting to find dependencies...")
		os.mkdir('TEMP_WORK_DIR')

		tarball_url = None
		for url in json['urls']:
			if url['url'].endswith('.tar.gz'):
				tarball_url = url['url']
				break
		assert tarball_url, "tarball not found"

		print("Downloading tarball...")
		tarball_path = 'TEMP_WORK_DIR/' + tarball_url.split('/')[-1]
		with requests.get(tarball_url, stream=True) as r:
			with open(tarball_path, 'wb') as fh:
				shutil.copyfileobj(r.raw, fh)

		print("Extracting...")
		subprocess.run(['tar', '-xzf', tarball_path, '-C', 'TEMP_WORK_DIR'])

		print("Hooking setup.py...")
		pkg_dir = 'TEMP_WORK_DIR/' + tarball_url.split('/')[-1].rpartition('.tar')[0]
		assert os.path.exists(pkg_dir + '/setup.py'), f"extracted setup.py not found in {pkg_dir}"
		deps = get_setuptools_deps(pkg_dir)

		pkg_data['setuptools_deps'] = deps
	except BaseException as e:
		print("Error determining package dependencies. YOU WILL HAVE TO MANUALLY SPECIFY RDEPENDS!")
		traceback.print_exception(e)
	finally:
		print("Cleaning up...")
		if os.path.exists('TEMP_WORK_DIR'):
			shutil.rmtree('TEMP_WORK_DIR')

	for dep in pkg_data['setuptools_deps']:
		pkg_data['dependencies'].append(translate_pip_dep(dep))

	return pkg_data

def fill_template(pkg_data: dict) -> str:
	ebuild = f'''
	# Ebuild for {pkg_data['pypi_name']}
	# Auto-generated by pipgen.py - TEST ME!
	EAPI=8

	DISTUTILS_USE_PEP517=setuptools
	PYTHON_COMPAT=( python3_{{8..11}} pypy3 )

	inherit distutils-r1

	DESCRIPTION="{pkg_data['description']}"
	HOMEPAGE="
		{pkg_data['home_url']}
		{pkg_data['pypi_url']}
	"

	MY_PN="{pkg_data['pypi_name']}"
	MY_P="${{MY_PN}}-${{PV}}"

	if [[ "${{PV}}" == *9999* ]]; then
		EGIT_REPO_URI="{pkg_data['git_url']}"
		inherit git-r3
	else
		SRC_URI="mirror://pypi/${{MY_P:0:1}}/${{MY_PN}}/${{MY_P}}.tar.gz"
		KEYWORDS="~alpha ~amd64 ~arm ~arm64 ~hppa ~ia64 ~loong ~mips ~ppc ~ppc64 ~riscv ~s390 ~sparc ~x86"
		S="${{WORKDIR}}/${{MY_P}}"
	fi

	LICENSE="{pkg_data['license']}"
	SLOT="0"
	IUSE=""
	'''
	ebuild = '\n'.join(line[1:] if line.startswith('\t') else line for line in ebuild.split('\n')) + "\n"
	ebuild += 'RDEPEND="\n'
	for pkg in pkg_data['dependencies']:
		ebuild += f"\t{pkg}[${{PYTHON_USEDEP}}]\n"
	ebuild += '"\n'

	ebuild += "# pkg_data:\n"
	for line in pprint.pformat(pkg_data).split('\n'):
		ebuild += f"# {line}\n"

	return ebuild

def __main__():
	parser = argparse.ArgumentParser(description='auto-generate ebuilds from pypi packages')
	parser.add_argument('name', help='pypi package name')
	parser.add_argument('version', nargs='?', default=None, help='package version (latest if unspecified)')
	parser.add_argument('-O', '--output', default=None, help='desired gentoo package name')

	args = parser.parse_args()

	pkg_name = args.output or 'dev-python/' + args.name.replace('_', '-').replace('.', '-')
	if not os.path.exists(pkg_name):
		os.makedirs(pkg_name)

	pkg_data = get_package((args.name + '/' + args.version) if args.version else args.name)
	with open(f'{pkg_name}/{pkg_name.partition("/")[2]}-{pkg_data["version"]}.ebuild', 'w') as fh:
		fh.write(fill_template(pkg_data))

if __name__ == '__main__':
	__main__()