Updated migrate to write TOML files

Europe/Sweden/ArchLife.toml

-[compatability]
-version = "v1"
-
-[ArchLife]
-tier = 1
-upstream = "Worldwide.Tier0"
-
-# In Gb/s
-bandwidth = 10
-
-IP = [
-	"89.190.3.198",
-	"2a03:7c00:11:25::198"
-]
-
-https = [
-	"https://archlinux.life/repo/"
-]
-
-http = [
-	"http://archlinux.life/repo/"
-]
-
-rsync = [
-	"rsync://archlinux.life/repo/"
-]
\ No newline at end of file

Europe/Sweden/EvilScientist.toml

-[compatability]
-version = "v1"
-
-[EvilScientist]
-tier = 2
-upstream = "Europe.Sweden.ArchLife"
-
-# In Gb/s
-bandwidth = 10
-
-https = [
-	"https://scientist.cloud"
-]
-
-http = [
-	"http://scientist.cloud"
-]
-
-rsync = [
-	"rsync://scientist.cloud"
-]
\ No newline at end of file

Europe/Sweden/Hvornum.toml

-[compatability]
-version = "v1"
-
-[Hvornum]
-tier = 2
-upstream = "Europe.Sweden.ArchLife"
-visible = true
-
-# In Gb/s
-bandwidth = 0.1
-
-https = [
-	"https://hvornum.se"
-]
-
-http = [
-	"http://hvornum.se"
-]
-
-rsync = [
-	"rsync://hvornum.se"
-]
\ No newline at end of file

Worldwide/Tier0.toml

-[compatability]
-version = "v1"
-
-[Tier0]
-tier = 0
-
-# In Gb/s
-bandwidth = 40
-
-https = [
-	"https://archlinux.org/repo/"
-]
-
-http = [
-	"http://archlinux.org/repo/"
-]
-
-rsync = [
-	"rsync://archlinux.org/repo/"
-]
\ No newline at end of file

arch_mirrors/cli/functions/migrate.py

-import tomllib
+import toml
 import psycopg2
 import pathlib
+import logging
 import pycountry
 import pycountry_convert as continent
 
 from ...database.postgresql import Database
+from ...output import log
 
 def country_to_continent(country_name):
 	country_alpha2 = continent.country_name_to_country_alpha2(country_name)
@@ -24,10 +26,14 @@ def migrate(args):
 		port = args.psql_port
 	)
 
-
+	DEBUG_MIRROR = 'lysator.liu.se'
 	if args.source == 'archweb':
-		for mirror in db.query("SELECT * FROM mirrors_mirror LIMIT 10", force_list=True):
-			#print(f"Mirror {mirror.get('name')}: {mirror}")
+		for mirror in db.query("SELECT * FROM mirrors_mirror", force_list=True):
+			if mirror['name'] == DEBUG_MIRROR:
+				log(f"Mirror {mirror.get('name')}: {mirror}")
+
+			if mirror['public'] == False:
+				log(f" -- Mirror {mirror['name']} is not public", fg="gray", level=logging.DEBUG)
 
 			# Create the v1 spec structure with sane defaults
 			mirror_spec = {
@@ -41,9 +47,10 @@ def migrate(args):
 					'http' : [],
 					'rsync' : [],
 					'ftp' : [],
-					'visible' : False,
+					'visible' : mirror['public'],
+					'active' : mirror['active'],
 					'upstream' : None,
-					'IP' : [],
+					'IP' : [row['ip'] for row in db.query("SELECT ip FROM mirrors_mirrorrsync WHERE mirror_id=%s", (mirror['id'], ), force_list=True)],
 					# The following should be auto-detected:
 					# 'iso' : False,
 					# 'images' : False,
@@ -59,20 +66,40 @@ def migrate(args):
 				mirror_spec[mirror['name']]['upstream'] = db.query("SELECT name FROM mirrors_mirror WHERE id=%s", (mirror['upstream_id'], ), force_list=True)[0]['name']
 
 			_continent = _country = _region = None
+
 			for url in db.query("SELECT * FROM mirrors_mirrorurl WHERE mirror_id=%s", (mirror['id'], ), force_list=True):
 				if _region is None:
 					_region = url.get('country', None)
 				elif url.get('country', None) is not None and _region != url['country']:
 					_region = 'Worldwide'
 
-				#print(f"  {url}")
+				if mirror['name'] == DEBUG_MIRROR:
+					log(f"  {url}", fg="gray", level=logging.DEBUG)
 
-			if _region:
-				_region = pycountry.countries.get(alpha_2=_region)
-				_country = _region.name
+				if url['url'].startswith('https://'):
+					mirror_spec[mirror['name']]['https'].append(url['url'])
+				elif url['url'].startswith('http://'):
+					mirror_spec[mirror['name']]['http'].append(url['url'])
+				elif url['url'].startswith('rsync://'):
+					mirror_spec[mirror['name']]['rsync'].append(url['url'])
+				elif url['url'].startswith('ftp://'):
+					mirror_spec[mirror['name']]['ftp'].append(url['url'])
+
+			if _region and _region != 'Worldwide':
+				_country_obj = pycountry.countries.get(alpha_2=_region)
+				_country = _country_obj.name
 				_continent = country_to_continent(_country)
 
-			mirror_path = pathlib.Path('./') / _continent / _country / f"{mirror['name']}.toml"
+				mirror_path = pathlib.Path('./') / _continent / _country / f"{mirror['name']}.toml"
+			elif _region == 'Worldwide':
+				mirror_path = pathlib.Path('./') / 'Worldwide' / f"{mirror['name']}.toml"
+			else:
+				mirror_path = pathlib.Path('./') / 'rsync_only' / f"{mirror['name']}.toml"
 
 			# Don't write them yet
-			print(mirror_path)
\ No newline at end of file
+			if mirror['name'] == DEBUG_MIRROR:
+				log(f"Saving to {mirror_path}", fg="gray", level=logging.DEBUG)
+
+			mirror_path.parent.mkdir(parents=True, exist_ok=True)
+			with mirror_path.open('w') as fh:
+				toml.dump(mirror_spec, fh)
\ No newline at end of file

arch_mirrors/database/postgresql.py

@@ -79,7 +79,7 @@ class Database:
 					# INSERT etc will trigger a rowcount (good), but won't have any results to return (good)
 					return True
 
-		return None
+		return [] if force_list else None
 
 class Transaction:
 	session :Database