Skip to content
Snippets Groups Projects

Added migrate, to migrate archweb postgresql to TOML files

Open Anton Hvornum requested to merge migration-job into main

This is just helper stuff in preparation for RFC !29.

Edited by Anton Hvornum

Merge request reports

Loading
Loading

Activity

Filter activity
  • Approvals
  • Assignees & reviewers
  • Comments (from bots)
  • Comments (from users)
  • Commits & branches
  • Edits
  • Labels
  • Lock status
  • Mentions
  • Merge request status
  • Tracking
  • Anton Hvornum added 1 commit

    added 1 commit

    • 80cccef7 - Added first draft of what a database->TOML would look like

    Compare with previous version

  • Anton Hvornum changed the description

    changed the description

  • Anton Hvornum added 1 commit

    added 1 commit

    • 037bea4e - Updated migrate to write TOML files

    Compare with previous version

  • Anton Hvornum changed the description

    changed the description

  • Author Developer

    Remaining question on how to organize the TOML files:

    • Multi-country mirrors should live where? If it's a truly global CND spanning all countries then ./Worldwide/ makes sense. But if it's only originating from say 3-4 countries then worldwide is misleading. And how would you include Germany + <that one worldwide that originates from germany>?
    • Should mirrors like https://armbrust.me/ have it's TOML in two folders, ./Europe/France/armbrust.me.toml where french URL's are listed and ./Europe/Finland/armbrust.me.toml where finnish URL's are listed? It multiplies the work creating TOML files, but it also adds fine granularity and makes it easier to set different speeds for different regions.
    Edited by Anton Hvornum
  • Author Developer

    In order to preserve the archweb data as-is, we'd have to do something along the lines of:

    [compatibility]
    version = "v1"
    
    [mirror]
    name = "armbrust.me"
    visible = true
    active = true
    upstream = "accum.se"
    tier = 2
    
    iso = true
    
    rsync_from = [
    	"127.0.0.1",
    	"::1"
    ]
    
    
    [country."FR".http."http://mirror.armbrust.me/archlinux/"]
    bandwidth = 100 #MBit
    active = false
    ivp4 = true
    ipv6 = false
    
    [country."FR".https."https://mirror.armbrust.me/archlinux/"]
    bandwidth = 100 #MBit
    active = false
    ivp4 = true
    ipv6 = false
    
    [country."FR".rsync."rsync://mirror.armbrust.me/archlinux/"]
    bandwidth = 100 #MBit
    active = false
    ivp4 = true
    ipv6 = false

    Which will translate to JSON:

    {
        "compatibility": {
            "version": "v1"
        },
        "mirror": {
            "name": "armbrust.me",
            "visible": true,
            "active": true,
            "upstream": "accum.se",
            "tier": 2,
            "iso": true,
            "rsync_from": [
                "127.0.0.1",
                "::1"
            ]
        },
        "country": {
            "FR": {
                "http": {
                    "http://mirror.armbrust.me/archlinux/": {
                        "bandwidth": 100,
                        "active": false,
                        "ivp4": true,
                        "ipv6": false
                    }
                },
                "https": {
                    "https://mirror.armbrust.me/archlinux/": {
                        "bandwidth": 100,
                        "active": false,
                        "ivp4": true,
                        "ipv6": false
                    }
                },
                "rsync": {
                    "rsync://mirror.armbrust.me/archlinux/": {
                        "bandwidth": 100,
                        "active": false,
                        "ivp4": true,
                        "ipv6": false
                    }
                }
            }
        }
    }

    I believe we can remove ISO as that should ideally be auto-detected anyway. And active and visible feels like duplicates of each other.

  • Author Developer

    An alternative would be:

    [compatibility]
    version = "v1"
    
    [mirror]
    name = "armbrust.me"
    visible = true
    active = true
    upstream = "accum.se"
    tier = 2
    
    iso = true
    
    rsync_from = [
    	"127.0.0.1",
    	"::1"
    ]
    
    [url.http."http://mirror.armbrust.me/archlinux/"]
    country = "FR"
    bandwidth = 100 #MBit
    active = false
    ivp4 = true
    ipv6 = false
    
    [url.https."https://mirror.armbrust.me/archlinux/"]
    country = "FR"
    bandwidth = 100 #MBit
    active = false
    ivp4 = true
    ipv6 = false
    
    [url.rsync."rsync://mirror.armbrust.me/archlinux/"]
    country = "FR"
    bandwidth = 100 #MBit
    active = false
    ivp4 = true
    ipv6 = false

    Where we don't group URL's by the country. Easier to write, but slightly harder to humanly distinguish which mirrors are grouped by a region.

    It translates to JSON:

    {
        "compatibility": {
            "version": "v1"
        },
        "mirror": {
            "name": "armbrust.me",
            "visible": true,
            "active": true,
            "upstream": "accum.se",
            "tier": 2,
            "iso": true,
            "rsync_from": [
                "127.0.0.1",
                "::1"
            ]
        },
        "url": {
            "http": {
                "http://mirror.armbrust.me/archlinux/": {
                    "country": "FR",
                    "bandwidth": 100,
                    "active": false,
                    "ivp4": true,
                    "ipv6": false
                }
            },
            "https": {
                "https://mirror.armbrust.me/archlinux/": {
                    "country": "FR",
                    "bandwidth": 100,
                    "active": false,
                    "ivp4": true,
                    "ipv6": false
                }
            },
            "rsync": {
                "rsync://mirror.armbrust.me/archlinux/": {
                    "country": "FR",
                    "bandwidth": 100,
                    "active": false,
                    "ivp4": true,
                    "ipv6": false
                }
            }
        }
    }
  • Author Developer

    Suggested final format:

    [compatibility]
    version = "v1"
    
    [mirror]
    name = "armbrust.me"
    visible = true
    active = true
    upstream = "accum.se"
    tier = 2
    
    iso = true
    
    rsync_from = [
    	"127.0.0.1",
    	"::1"
    ]
    
    [url."http://mirror.armbrust.me/archlinux/"]
    country = "FR"
    bandwidth = 100 #MBit
    active = false
    ivp4 = true
    ipv6 = false
    
    [url."https://mirror.armbrust.me/archlinux/"]
    country = "FR"
    bandwidth = 100 #MBit
    active = false
    ivp4 = true
    ipv6 = false
    
    [url."rsync://mirror.armbrust.me/archlinux/"]
    country = "FR"
    bandwidth = 100 #MBit
    active = false
    ivp4 = true
    ipv6 = false
  • Anton Hvornum mentioned in merge request rfcs!29 (merged)

    mentioned in merge request rfcs!29 (merged)

Please register or sign in to reply
Loading