Commit bac38edd authored by Kevin Morris's avatar Kevin Morris
Browse files

[db] fix schema and migration for case insensitivity



Some of the columns that were changed still want to be
case insensitive. Good thing our tables have nice
separation.
Signed-off-by: Kevin Morris's avatarKevin Morris <kevr@0cost.org>
parent b1121dc6
...@@ -10,6 +10,10 @@ from sqlalchemy import CHAR, TIMESTAMP, Column, ForeignKey, Index, MetaData, Str ...@@ -10,6 +10,10 @@ from sqlalchemy import CHAR, TIMESTAMP, Column, ForeignKey, Index, MetaData, Str
from sqlalchemy.dialects.mysql import BIGINT, DECIMAL, INTEGER, TINYINT from sqlalchemy.dialects.mysql import BIGINT, DECIMAL, INTEGER, TINYINT
from sqlalchemy.ext.compiler import compiles from sqlalchemy.ext.compiler import compiles
import aurweb.config
db_backend = aurweb.config.get("database", "backend")
@compiles(TINYINT, 'sqlite') @compiles(TINYINT, 'sqlite')
def compile_tinyint_sqlite(type_, compiler, **kw): def compile_tinyint_sqlite(type_, compiler, **kw):
...@@ -35,7 +39,9 @@ AccountTypes = Table( ...@@ -35,7 +39,9 @@ AccountTypes = Table(
'AccountTypes', metadata, 'AccountTypes', metadata,
Column('ID', TINYINT(unsigned=True), primary_key=True), Column('ID', TINYINT(unsigned=True), primary_key=True),
Column('AccountType', String(32), nullable=False, server_default=text("''")), Column('AccountType', String(32), nullable=False, server_default=text("''")),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin' mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci'
) )
...@@ -69,7 +75,9 @@ Users = Table( ...@@ -69,7 +75,9 @@ Users = Table(
Column('OwnershipNotify', TINYINT(1), nullable=False, server_default=text("1")), Column('OwnershipNotify', TINYINT(1), nullable=False, server_default=text("1")),
Column('SSOAccountID', String(255), nullable=True, unique=True), Column('SSOAccountID', String(255), nullable=True, unique=True),
Index('UsersAccountTypeID', 'AccountTypeID'), Index('UsersAccountTypeID', 'AccountTypeID'),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -113,7 +121,9 @@ PackageBases = Table( ...@@ -113,7 +121,9 @@ PackageBases = Table(
Index('BasesNumVotes', 'NumVotes'), Index('BasesNumVotes', 'NumVotes'),
Index('BasesPackagerUID', 'PackagerUID'), Index('BasesPackagerUID', 'PackagerUID'),
Index('BasesSubmitterUID', 'SubmitterUID'), Index('BasesSubmitterUID', 'SubmitterUID'),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -122,7 +132,9 @@ PackageKeywords = Table( ...@@ -122,7 +132,9 @@ PackageKeywords = Table(
'PackageKeywords', metadata, 'PackageKeywords', metadata,
Column('PackageBaseID', ForeignKey('PackageBases.ID', ondelete='CASCADE'), primary_key=True, nullable=False), Column('PackageBaseID', ForeignKey('PackageBases.ID', ondelete='CASCADE'), primary_key=True, nullable=False),
Column('Keyword', String(255), primary_key=True, nullable=False, server_default=text("''")), Column('Keyword', String(255), primary_key=True, nullable=False, server_default=text("''")),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -135,7 +147,9 @@ Packages = Table( ...@@ -135,7 +147,9 @@ Packages = Table(
Column('Version', String(255), nullable=False, server_default=text("''")), Column('Version', String(255), nullable=False, server_default=text("''")),
Column('Description', String(255)), Column('Description', String(255)),
Column('URL', String(8000)), Column('URL', String(8000)),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -144,7 +158,9 @@ Licenses = Table( ...@@ -144,7 +158,9 @@ Licenses = Table(
'Licenses', metadata, 'Licenses', metadata,
Column('ID', INTEGER(unsigned=True), primary_key=True), Column('ID', INTEGER(unsigned=True), primary_key=True),
Column('Name', String(255), nullable=False, unique=True), Column('Name', String(255), nullable=False, unique=True),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -162,7 +178,9 @@ Groups = Table( ...@@ -162,7 +178,9 @@ Groups = Table(
'Groups', metadata, 'Groups', metadata,
Column('ID', INTEGER(unsigned=True), primary_key=True), Column('ID', INTEGER(unsigned=True), primary_key=True),
Column('Name', String(255), nullable=False, unique=True), Column('Name', String(255), nullable=False, unique=True),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -180,7 +198,9 @@ DependencyTypes = Table( ...@@ -180,7 +198,9 @@ DependencyTypes = Table(
'DependencyTypes', metadata, 'DependencyTypes', metadata,
Column('ID', TINYINT(unsigned=True), primary_key=True), Column('ID', TINYINT(unsigned=True), primary_key=True),
Column('Name', String(32), nullable=False, server_default=text("''")), Column('Name', String(32), nullable=False, server_default=text("''")),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -195,7 +215,9 @@ PackageDepends = Table( ...@@ -195,7 +215,9 @@ PackageDepends = Table(
Column('DepArch', String(255)), Column('DepArch', String(255)),
Index('DependsDepName', 'DepName'), Index('DependsDepName', 'DepName'),
Index('DependsPackageID', 'PackageID'), Index('DependsPackageID', 'PackageID'),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -204,7 +226,9 @@ RelationTypes = Table( ...@@ -204,7 +226,9 @@ RelationTypes = Table(
'RelationTypes', metadata, 'RelationTypes', metadata,
Column('ID', TINYINT(unsigned=True), primary_key=True), Column('ID', TINYINT(unsigned=True), primary_key=True),
Column('Name', String(32), nullable=False, server_default=text("''")), Column('Name', String(32), nullable=False, server_default=text("''")),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -218,7 +242,9 @@ PackageRelations = Table( ...@@ -218,7 +242,9 @@ PackageRelations = Table(
Column('RelArch', String(255)), Column('RelArch', String(255)),
Index('RelationsPackageID', 'PackageID'), Index('RelationsPackageID', 'PackageID'),
Index('RelationsRelName', 'RelName'), Index('RelationsRelName', 'RelName'),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -229,7 +255,9 @@ PackageSources = Table( ...@@ -229,7 +255,9 @@ PackageSources = Table(
Column('Source', String(8000), nullable=False, server_default=text("'/dev/null'")), Column('Source', String(8000), nullable=False, server_default=text("'/dev/null'")),
Column('SourceArch', String(255)), Column('SourceArch', String(255)),
Index('SourcesPackageID', 'PackageID'), Index('SourcesPackageID', 'PackageID'),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -262,7 +290,9 @@ PackageComments = Table( ...@@ -262,7 +290,9 @@ PackageComments = Table(
Column('PinnedTS', BIGINT(unsigned=True), nullable=False, server_default=text("0")), Column('PinnedTS', BIGINT(unsigned=True), nullable=False, server_default=text("0")),
Index('CommentsPackageBaseID', 'PackageBaseID'), Index('CommentsPackageBaseID', 'PackageBaseID'),
Index('CommentsUsersID', 'UsersID'), Index('CommentsUsersID', 'UsersID'),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -293,7 +323,9 @@ PackageBlacklist = Table( ...@@ -293,7 +323,9 @@ PackageBlacklist = Table(
'PackageBlacklist', metadata, 'PackageBlacklist', metadata,
Column('ID', INTEGER(unsigned=True), primary_key=True), Column('ID', INTEGER(unsigned=True), primary_key=True),
Column('Name', String(64), nullable=False, unique=True), Column('Name', String(64), nullable=False, unique=True),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -314,7 +346,9 @@ RequestTypes = Table( ...@@ -314,7 +346,9 @@ RequestTypes = Table(
'RequestTypes', metadata, 'RequestTypes', metadata,
Column('ID', TINYINT(unsigned=True), primary_key=True), Column('ID', TINYINT(unsigned=True), primary_key=True),
Column('Name', String(32), nullable=False, server_default=text("''")), Column('Name', String(32), nullable=False, server_default=text("''")),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -335,7 +369,9 @@ PackageRequests = Table( ...@@ -335,7 +369,9 @@ PackageRequests = Table(
Column('Status', TINYINT(unsigned=True), nullable=False, server_default=text("0")), Column('Status', TINYINT(unsigned=True), nullable=False, server_default=text("0")),
Index('RequestsPackageBaseID', 'PackageBaseID'), Index('RequestsPackageBaseID', 'PackageBaseID'),
Index('RequestsUsersID', 'UsersID'), Index('RequestsUsersID', 'UsersID'),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -371,7 +407,9 @@ Bans = Table( ...@@ -371,7 +407,9 @@ Bans = Table(
'Bans', metadata, 'Bans', metadata,
Column('IPAddress', String(45), primary_key=True), Column('IPAddress', String(45), primary_key=True),
Column('BanTS', TIMESTAMP, nullable=False), Column('BanTS', TIMESTAMP, nullable=False),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -382,7 +420,9 @@ Terms = Table( ...@@ -382,7 +420,9 @@ Terms = Table(
Column('Description', String(255), nullable=False), Column('Description', String(255), nullable=False),
Column('URL', String(8000), nullable=False), Column('URL', String(8000), nullable=False),
Column('Revision', INTEGER(unsigned=True), nullable=False, server_default=text("1")), Column('Revision', INTEGER(unsigned=True), nullable=False, server_default=text("1")),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -403,5 +443,7 @@ ApiRateLimit = Table( ...@@ -403,5 +443,7 @@ ApiRateLimit = Table(
Column('Requests', INTEGER(11), nullable=False), Column('Requests', INTEGER(11), nullable=False),
Column('WindowStart', BIGINT(20), nullable=False), Column('WindowStart', BIGINT(20), nullable=False),
Index('ApiRateLimitWindowStart', 'WindowStart'), Index('ApiRateLimitWindowStart', 'WindowStart'),
mysql_engine='InnoDB', mysql_charset='utf8mb4', mysql_collate='utf8mb4_bin', mysql_engine='InnoDB',
mysql_charset='utf8mb4',
mysql_collate='utf8mb4_general_ci',
) )
...@@ -7,6 +7,8 @@ Create Date: 2021-05-17 14:23:00.008479 ...@@ -7,6 +7,8 @@ Create Date: 2021-05-17 14:23:00.008479
""" """
from alembic import op from alembic import op
import aurweb.config
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '56e2ce8e2ffa' revision = '56e2ce8e2ffa'
down_revision = 'ef39fcd6e1cd' down_revision = 'ef39fcd6e1cd'
...@@ -14,10 +16,31 @@ branch_labels = None ...@@ -14,10 +16,31 @@ branch_labels = None
depends_on = None depends_on = None
# Tables affected by charset/collate change # Tables affected by charset/collate change
tables = ['AccountTypes', 'ApiRateLimit', 'Bans', 'DependencyTypes', 'Groups', 'Licenses', 'OfficialProviders', tables = [
'PackageBases', 'PackageBlacklist', 'PackageComments', 'PackageDepends', 'PackageKeywords', ('AccountTypes', 'utf8mb4', 'utf8mb4_general_ci'),
'PackageRelations', 'PackageRequests', 'PackageSources', 'Packages', 'RelationTypes', 'RequestTypes', ('ApiRateLimit', 'utf8mb4', 'utf8mb4_general_ci'),
'SSHPubKeys', 'Sessions', 'TU_VoteInfo', 'Terms', 'Users'] ('Bans', 'utf8mb4', 'utf8mb4_general_ci'),
('DependencyTypes', 'utf8mb4', 'utf8mb4_general_ci'),
('Groups', 'utf8mb4', 'utf8mb4_general_ci'),
('Licenses', 'utf8mb4', 'utf8mb4_general_ci'),
('OfficialProviders', 'utf8mb4', 'utf8mb4_bin'),
('PackageBases', 'utf8mb4', 'utf8mb4_general_ci'),
('PackageBlacklist', 'utf8mb4', 'utf8mb4_general_ci'),
('PackageComments', 'utf8mb4', 'utf8mb4_general_ci'),
('PackageDepends', 'utf8mb4', 'utf8mb4_general_ci'),
('PackageKeywords', 'utf8mb4', 'utf8mb4_general_ci'),
('PackageRelations', 'utf8mb4', 'utf8mb4_general_ci'),
('PackageRequests', 'utf8mb4', 'utf8mb4_general_ci'),
('PackageSources', 'utf8mb4', 'utf8mb4_general_ci'),
('Packages', 'utf8mb4', 'utf8mb4_general_ci'),
('RelationTypes', 'utf8mb4', 'utf8mb4_general_ci'),
('RequestTypes', 'utf8mb4', 'utf8mb4_general_ci'),
('SSHPubKeys', 'utf8mb4', 'utf8mb4_bin'),
('Sessions', 'utf8mb4', 'utf8mb4_bin'),
('TU_VoteInfo', 'utf8mb4', 'utf8mb4_bin'),
('Terms', 'utf8mb4', 'utf8mb4_general_ci'),
('Users', 'utf8mb4', 'utf8mb4_general_ci')
]
# Indexes affected by charset/collate change # Indexes affected by charset/collate change
# Map of Unique Indexes key = index_name, value = [table_name, column1, column2] # Map of Unique Indexes key = index_name, value = [table_name, column1, column2]
...@@ -27,9 +50,7 @@ indexes = {'ProviderNameProvides': ['OfficialProviders', 'Name', 'Provides']} ...@@ -27,9 +50,7 @@ indexes = {'ProviderNameProvides': ['OfficialProviders', 'Name', 'Provides']}
src_charset = "utf8" src_charset = "utf8"
src_collate = "utf8_general_ci" src_collate = "utf8_general_ci"
# Destination charset/collation, after this migration is run. db_backend = aurweb.config.get("database", "backend")
dst_charset = "utf8mb4"
dst_collate = "utf8mb4_bin"
def rebuild_unique_indexes_with_str_cols(): def rebuild_unique_indexes_with_str_cols():
...@@ -53,11 +74,12 @@ def do_all(iterable, fn): ...@@ -53,11 +74,12 @@ def do_all(iterable, fn):
def upgrade(): def upgrade():
def op_execute(table): def op_execute(table_meta):
table, charset, collate = table_meta
sql = f""" sql = f"""
ALTER TABLE {table} ALTER TABLE {table}
CONVERT TO CHARACTER SET {dst_charset} CONVERT TO CHARACTER SET {charset}
COLLATE {dst_collate} COLLATE {collate}
""" """
op.execute(sql) op.execute(sql)
...@@ -66,7 +88,11 @@ COLLATE {dst_collate} ...@@ -66,7 +88,11 @@ COLLATE {dst_collate}
def downgrade(): def downgrade():
def op_execute(table): if db_backend == "sqlite":
return None
def op_execute(table_meta):
table, charset, collate = table_meta
sql = f""" sql = f"""
ALTER TABLE {table} ALTER TABLE {table}
CONVERT TO CHARACTER SET {src_charset} CONVERT TO CHARACTER SET {src_charset}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment