diff --git a/Database-Converter/README.md b/Database-Converter/README.md new file mode 100644 index 0000000..a5b0889 --- /dev/null +++ b/Database-Converter/README.md @@ -0,0 +1,279 @@ +# 🔄 Relational Database to MongoDB Converter + +A powerful Streamlit application that converts tables from relational databases (MySQL, PostgreSQL, SQLite, SQL Server) to MongoDB with an intuitive web interface. + +## ✨ Features + +- **Multi-Database Support**: Connect to MySQL, PostgreSQL, SQLite, and SQL Server +- **Interactive UI**: User-friendly web interface built with Streamlit +- **Table Selection**: Browse and select specific tables to migrate +- **Data Type Mapping**: Automatic conversion of SQL data types to MongoDB-compatible types +- **Batch Processing**: Efficient migration of large datasets with configurable batch sizes +- **Progress Tracking**: Real-time progress indicators during migration +- **Table Preview**: View table structure and sample data before migration +- **Migration Logging**: Detailed logs of all migration operations +- **Export Logs**: Download migration logs in JSON format + +## 📋 Prerequisites + +- Python 3.8 or higher +- pip (Python package manager) +- Access to source relational database +- MongoDB instance (local or remote) + +## 🚀 Installation + +1. **Navigate to the Database-Converter directory:** + ```bash + cd Database-Converter + ``` + +2. **Install pip (if not already installed):** + ```bash + python3 -m ensurepip --upgrade + ``` + +3. **Install required dependencies:** + ```bash + pip install -r requirements.txt + ``` + + **Note for SQL Server users:** You may need to install ODBC Driver 17 for SQL Server: + - **Linux (Amazon Linux/RHEL/CentOS):** + ```bash + sudo dnf install -y unixODBC-devel + # Follow Microsoft's guide for ODBC Driver installation + ``` + - **Ubuntu/Debian:** + ```bash + sudo apt-get install unixodbc-dev + ``` + - **macOS:** + ```bash + brew install unixodbc + ``` + +## 🎯 Usage + +### 1. Start the Application + +```bash +streamlit run streamlit_db_converter.py +``` + +The application will open in your default web browser at `http://localhost:8501` + +### 2. Configure Source Database + +In the sidebar, select your source database type and enter connection details: + +**For MySQL/PostgreSQL/SQL Server:** +- Host (e.g., `localhost`) +- Port (MySQL: 3306, PostgreSQL: 5432, SQL Server: 1433) +- Database Name +- Username +- Password + +**For SQLite:** +- File Path (e.g., `database.db`) + +Click **"Connect to Source DB"** to establish connection. + +### 3. Configure MongoDB + +Enter MongoDB connection details: +- Host (e.g., `localhost`) +- Port (default: 27017) +- Database Name +- Optional: Username, Password, and Auth Database + +Click **"Connect to MongoDB"** to establish connection. + +### 4. Select Tables + +- Browse available tables from your source database +- Use checkboxes to select tables for migration +- Use "Select All" / "Deselect All" for bulk selection +- Preview table structure and sample data + +### 5. Configure Migration + +- Set batch size (default: 1000 rows per batch) +- Larger batch sizes = faster migration but more memory usage +- Smaller batch sizes = slower but more stable for large tables + +### 6. Start Migration + +- Click **"🚀 Start Migration"** to begin the process +- Monitor real-time progress for each table +- View migration summary upon completion +- Download migration logs for record-keeping + +## 📊 Data Type Mapping + +The application automatically maps SQL data types to MongoDB-compatible types: + +| SQL Type | MongoDB Type | +|----------|--------------| +| INT, BIGINT, SERIAL | integer | +| FLOAT, DOUBLE, DECIMAL | float | +| BOOLEAN, BIT | boolean | +| DATE, DATETIME, TIMESTAMP | datetime | +| JSON, JSONB | json | +| BLOB, BINARY | binary | +| VARCHAR, TEXT, CHAR | string | + +## 🔧 Configuration Options + +### Batch Size +- **Small (100-500)**: For tables with large rows or limited memory +- **Medium (500-2000)**: Balanced performance (recommended) +- **Large (2000-10000)**: For fast migration of simple tables + +### Authentication +- **Source DB**: Always required (except SQLite) +- **MongoDB**: Optional, enable if your MongoDB requires authentication + +## 📝 Examples + +### Example 1: MySQL to MongoDB + +``` +Source Database: +- Type: MySQL +- Host: localhost +- Port: 3306 +- Database: myapp_db +- Username: root +- Password: **** + +Target MongoDB: +- Host: localhost +- Port: 27017 +- Database: myapp_mongo +- No authentication + +Tables: users, products, orders +Batch Size: 1000 +``` + +### Example 2: PostgreSQL to MongoDB (with Auth) + +``` +Source Database: +- Type: PostgreSQL +- Host: db.example.com +- Port: 5432 +- Database: production +- Username: admin +- Password: **** + +Target MongoDB: +- Host: mongo.example.com +- Port: 27017 +- Database: production_mongo +- Username: mongo_admin +- Password: **** +- Auth DB: admin + +Tables: customers, transactions +Batch Size: 2000 +``` + +### Example 3: SQLite to MongoDB + +``` +Source Database: +- Type: SQLite +- File Path: /path/to/database.db + +Target MongoDB: +- Host: localhost +- Port: 27017 +- Database: sqlite_migration +- No authentication + +Tables: all tables +Batch Size: 1000 +``` + +## 🐛 Troubleshooting + +### Connection Issues + +**Problem**: Cannot connect to source database +- Verify host, port, and credentials +- Check if database server is running +- Ensure firewall allows connections +- For remote databases, check network connectivity + +**Problem**: Cannot connect to MongoDB +- Verify MongoDB is running: `systemctl status mongod` +- Check MongoDB port is accessible +- Verify authentication credentials if enabled + +### Migration Issues + +**Problem**: Migration fails for specific tables +- Check table permissions +- Verify table is not locked +- Review error message in migration log +- Try smaller batch size + +**Problem**: Out of memory errors +- Reduce batch size +- Close other applications +- Migrate tables one at a time + +### Data Type Issues + +**Problem**: Data not converting correctly +- Check the data type mapping table +- Review sample data before migration +- Some complex types may need manual handling + +## 🔒 Security Notes + +- Never commit credentials to version control +- Use environment variables for sensitive data +- Ensure secure connections (SSL/TLS) for production +- Limit database user permissions to required tables only +- Regularly backup data before migration + +## 📚 Additional Resources + +- [Streamlit Documentation](https://docs.streamlit.io/) +- [SQLAlchemy Documentation](https://docs.sqlalchemy.org/) +- [PyMongo Documentation](https://pymongo.readthedocs.io/) +- [MongoDB Data Modeling](https://docs.mongodb.com/manual/core/data-modeling-introduction/) + +## 🤝 Contributing + +Contributions are welcome! Please feel free to submit issues or pull requests. + +## 📄 License + +This project is part of the 100 Plus Python Coding Problems repository. + +## 💡 Tips + +1. **Test First**: Always test with a small subset of data first +2. **Backup**: Backup your databases before migration +3. **Monitor**: Watch memory usage during large migrations +4. **Verify**: Check data integrity after migration +5. **Indexes**: Remember to create indexes in MongoDB after migration +6. **Schema Design**: Consider MongoDB schema design best practices + +## 🎓 Learning Resources + +This tool demonstrates: +- Database connectivity with SQLAlchemy +- MongoDB operations with PyMongo +- Interactive web apps with Streamlit +- Data transformation with Pandas +- Error handling and logging +- Batch processing techniques + +--- + +**Happy Migrating! 🚀** diff --git a/Database-Converter/requirements.txt b/Database-Converter/requirements.txt new file mode 100644 index 0000000..b5fe2d0 --- /dev/null +++ b/Database-Converter/requirements.txt @@ -0,0 +1,20 @@ +# Streamlit DB to MongoDB Converter - Dependencies + +# Core framework +streamlit>=1.28.0 + +# Database connectors +pymongo>=4.5.0 +sqlalchemy>=2.0.0 + +# SQL database drivers +mysql-connector-python>=8.1.0 +psycopg2-binary>=2.9.7 +pyodbc>=5.0.0 + +# Data processing +pandas>=2.0.0 +numpy>=1.24.0 + +# Utilities +python-dateutil>=2.8.2 diff --git a/Database-Converter/streamlit_db_converter.py b/Database-Converter/streamlit_db_converter.py new file mode 100644 index 0000000..0cf6207 --- /dev/null +++ b/Database-Converter/streamlit_db_converter.py @@ -0,0 +1,413 @@ +""" +Streamlit Application: Relational Database to MongoDB Converter +Supports MySQL, PostgreSQL, SQLite, and SQL Server +""" + +import streamlit as st +import pandas as pd +from sqlalchemy import create_engine, inspect, MetaData, Table, select +from pymongo import MongoClient +from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError +import traceback +from datetime import datetime +import json + +# Page configuration +st.set_page_config( + page_title="DB to MongoDB Converter", + page_icon="🔄", + layout="wide", + initial_sidebar_state="expanded" +) + +# Initialize session state +if 'source_engine' not in st.session_state: + st.session_state.source_engine = None +if 'mongo_client' not in st.session_state: + st.session_state.mongo_client = None +if 'selected_tables' not in st.session_state: + st.session_state.selected_tables = [] +if 'migration_log' not in st.session_state: + st.session_state.migration_log = [] + + +def create_sql_connection(db_type, host, port, database, username, password, sqlite_path=None): + """Create SQLAlchemy engine for different database types""" + try: + if db_type == "MySQL": + connection_string = f"mysql+mysqlconnector://{username}:{password}@{host}:{port}/{database}" + elif db_type == "PostgreSQL": + connection_string = f"postgresql+psycopg2://{username}:{password}@{host}:{port}/{database}" + elif db_type == "SQLite": + connection_string = f"sqlite:///{sqlite_path}" + elif db_type == "SQL Server": + connection_string = f"mssql+pyodbc://{username}:{password}@{host}:{port}/{database}?driver=ODBC+Driver+17+for+SQL+Server" + else: + return None, "Unsupported database type" + + engine = create_engine(connection_string) + # Test connection + with engine.connect() as conn: + pass + return engine, None + except Exception as e: + return None, str(e) + + +def create_mongo_connection(host, port, database, username=None, password=None, auth_db="admin"): + """Create MongoDB connection""" + try: + if username and password: + connection_string = f"mongodb://{username}:{password}@{host}:{port}/{database}?authSource={auth_db}" + else: + connection_string = f"mongodb://{host}:{port}/" + + client = MongoClient(connection_string, serverSelectionTimeoutMS=5000) + # Test connection + client.admin.command('ping') + return client, None + except (ConnectionFailure, ServerSelectionTimeoutError) as e: + return None, str(e) + except Exception as e: + return None, str(e) + + +def get_table_list(engine): + """Get list of tables from the database""" + try: + inspector = inspect(engine) + return inspector.get_table_names() + except Exception as e: + st.error(f"Error fetching tables: {str(e)}") + return [] + + +def get_table_info(engine, table_name): + """Get column information for a table""" + try: + inspector = inspect(engine) + columns = inspector.get_columns(table_name) + return columns + except Exception as e: + st.error(f"Error fetching table info: {str(e)}") + return [] + + +def convert_sql_to_mongo_type(sql_type): + """Map SQL data types to MongoDB-friendly types""" + sql_type_lower = str(sql_type).lower() + + if 'int' in sql_type_lower or 'serial' in sql_type_lower: + return 'integer' + elif 'float' in sql_type_lower or 'double' in sql_type_lower or 'decimal' in sql_type_lower or 'numeric' in sql_type_lower: + return 'float' + elif 'bool' in sql_type_lower or 'bit' in sql_type_lower: + return 'boolean' + elif 'date' in sql_type_lower or 'time' in sql_type_lower: + return 'datetime' + elif 'json' in sql_type_lower: + return 'json' + elif 'blob' in sql_type_lower or 'binary' in sql_type_lower: + return 'binary' + else: + return 'string' + + +def migrate_table(engine, mongo_db, table_name, batch_size=1000): + """Migrate a single table from SQL to MongoDB""" + try: + # Read data from SQL table + metadata = MetaData() + table = Table(table_name, metadata, autoload_with=engine) + + with engine.connect() as conn: + # Get total count + count_query = select(table) + result = conn.execute(count_query) + rows = result.fetchall() + total_rows = len(rows) + + if total_rows == 0: + return True, "Table is empty", 0 + + # Convert to DataFrame + df = pd.DataFrame(rows, columns=result.keys()) + + # Convert DataFrame to list of dictionaries + records = df.to_dict('records') + + # Handle data type conversions + for record in records: + for key, value in record.items(): + # Convert pandas/numpy types to Python native types + if pd.isna(value): + record[key] = None + elif isinstance(value, (pd.Timestamp, datetime)): + record[key] = value + elif hasattr(value, 'item'): # numpy types + record[key] = value.item() + + # Insert into MongoDB in batches + collection = mongo_db[table_name] + + progress_bar = st.progress(0) + status_text = st.empty() + + for i in range(0, len(records), batch_size): + batch = records[i:i + batch_size] + collection.insert_many(batch) + + progress = min((i + batch_size) / total_rows, 1.0) + progress_bar.progress(progress) + status_text.text(f"Migrated {min(i + batch_size, total_rows)}/{total_rows} rows") + + progress_bar.empty() + status_text.empty() + + return True, f"Successfully migrated {total_rows} rows", total_rows + + except Exception as e: + return False, f"Error: {str(e)}\n{traceback.format_exc()}", 0 + + +def main(): + st.title("🔄 Relational Database to MongoDB Converter") + st.markdown("Convert tables from MySQL, PostgreSQL, SQLite, or SQL Server to MongoDB") + + # Sidebar for connections + with st.sidebar: + st.header("⚙️ Configuration") + + # Source Database Configuration + st.subheader("1️⃣ Source Database") + db_type = st.selectbox( + "Database Type", + ["MySQL", "PostgreSQL", "SQLite", "SQL Server"] + ) + + if db_type == "SQLite": + sqlite_path = st.text_input("SQLite File Path", value="database.db") + if st.button("Connect to SQLite"): + engine, error = create_sql_connection(db_type, None, None, None, None, None, sqlite_path) + if error: + st.error(f"Connection failed: {error}") + else: + st.session_state.source_engine = engine + st.success("✅ Connected to SQLite!") + else: + col1, col2 = st.columns(2) + with col1: + host = st.text_input("Host", value="localhost") + with col2: + port = st.number_input("Port", value=3306 if db_type == "MySQL" else 5432, step=1) + + database = st.text_input("Database Name") + username = st.text_input("Username") + password = st.text_input("Password", type="password") + + if st.button("Connect to Source DB"): + if not all([host, database, username]): + st.error("Please fill in all required fields") + else: + engine, error = create_sql_connection(db_type, host, port, database, username, password) + if error: + st.error(f"Connection failed: {error}") + else: + st.session_state.source_engine = engine + st.success("✅ Connected to source database!") + + st.divider() + + # MongoDB Configuration + st.subheader("2️⃣ Target MongoDB") + mongo_host = st.text_input("MongoDB Host", value="localhost", key="mongo_host") + mongo_port = st.number_input("MongoDB Port", value=27017, step=1, key="mongo_port") + mongo_database = st.text_input("MongoDB Database Name", key="mongo_db") + + use_auth = st.checkbox("Use Authentication") + if use_auth: + mongo_username = st.text_input("MongoDB Username", key="mongo_user") + mongo_password = st.text_input("MongoDB Password", type="password", key="mongo_pass") + mongo_auth_db = st.text_input("Auth Database", value="admin", key="mongo_auth") + else: + mongo_username = None + mongo_password = None + mongo_auth_db = "admin" + + if st.button("Connect to MongoDB"): + if not mongo_database: + st.error("Please enter MongoDB database name") + else: + client, error = create_mongo_connection( + mongo_host, mongo_port, mongo_database, + mongo_username, mongo_password, mongo_auth_db + ) + if error: + st.error(f"Connection failed: {error}") + else: + st.session_state.mongo_client = client + st.success("✅ Connected to MongoDB!") + + # Main content area + if st.session_state.source_engine is None: + st.info("👈 Please connect to a source database using the sidebar") + return + + if st.session_state.mongo_client is None: + st.info("👈 Please connect to MongoDB using the sidebar") + return + + # Table selection and migration + st.header("📊 Select Tables to Migrate") + + tables = get_table_list(st.session_state.source_engine) + + if not tables: + st.warning("No tables found in the source database") + return + + # Display tables with checkboxes + col1, col2 = st.columns([2, 1]) + + with col1: + st.subheader(f"Available Tables ({len(tables)})") + + # Select all / Deselect all + col_a, col_b = st.columns(2) + with col_a: + if st.button("Select All"): + st.session_state.selected_tables = tables.copy() + with col_b: + if st.button("Deselect All"): + st.session_state.selected_tables = [] + + # Table selection + selected_tables = [] + for table in tables: + if st.checkbox(table, value=table in st.session_state.selected_tables, key=f"table_{table}"): + selected_tables.append(table) + + st.session_state.selected_tables = selected_tables + + with col2: + st.subheader("Migration Settings") + batch_size = st.number_input("Batch Size", min_value=100, max_value=10000, value=1000, step=100) + + if st.session_state.selected_tables: + st.info(f"Selected: {len(st.session_state.selected_tables)} table(s)") + + # Table preview + if st.session_state.selected_tables: + st.divider() + st.subheader("📋 Table Preview") + + preview_table = st.selectbox("Select table to preview", st.session_state.selected_tables) + + if preview_table: + col_info = get_table_info(st.session_state.source_engine, preview_table) + + if col_info: + st.write("**Column Information:**") + col_df = pd.DataFrame([ + { + "Column": col['name'], + "SQL Type": str(col['type']), + "MongoDB Type": convert_sql_to_mongo_type(col['type']), + "Nullable": col['nullable'] + } + for col in col_info + ]) + st.dataframe(col_df, use_container_width=True) + + # Show sample data + try: + sample_query = f"SELECT * FROM {preview_table} LIMIT 5" + sample_df = pd.read_sql(sample_query, st.session_state.source_engine) + st.write("**Sample Data (first 5 rows):**") + st.dataframe(sample_df, use_container_width=True) + except Exception as e: + st.warning(f"Could not fetch sample data: {str(e)}") + + # Migration button + st.divider() + + if st.session_state.selected_tables: + col1, col2, col3 = st.columns([1, 1, 2]) + + with col1: + if st.button("🚀 Start Migration", type="primary", use_container_width=True): + mongo_db = st.session_state.mongo_client[mongo_database] + + st.header("📈 Migration Progress") + + success_count = 0 + failed_count = 0 + total_rows = 0 + + for table in st.session_state.selected_tables: + st.subheader(f"Migrating: {table}") + + success, message, rows = migrate_table( + st.session_state.source_engine, + mongo_db, + table, + batch_size + ) + + if success: + st.success(f"✅ {table}: {message}") + success_count += 1 + total_rows += rows + else: + st.error(f"❌ {table}: {message}") + failed_count += 1 + + # Log migration + st.session_state.migration_log.append({ + "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "table": table, + "status": "Success" if success else "Failed", + "rows": rows, + "message": message + }) + + # Summary + st.divider() + st.header("📊 Migration Summary") + + col1, col2, col3 = st.columns(3) + with col1: + st.metric("✅ Successful", success_count) + with col2: + st.metric("❌ Failed", failed_count) + with col3: + st.metric("📝 Total Rows", total_rows) + + if success_count > 0: + st.balloons() + + with col2: + if st.button("🗑️ Clear Selection", use_container_width=True): + st.session_state.selected_tables = [] + st.rerun() + + # Migration log + if st.session_state.migration_log: + st.divider() + st.header("📜 Migration Log") + + log_df = pd.DataFrame(st.session_state.migration_log) + st.dataframe(log_df, use_container_width=True) + + # Download log + log_json = json.dumps(st.session_state.migration_log, indent=2) + st.download_button( + label="📥 Download Log (JSON)", + data=log_json, + file_name=f"migration_log_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json", + mime="application/json" + ) + + +if __name__ == "__main__": + main() diff --git a/trans b/trans new file mode 160000 index 0000000..853ff7a --- /dev/null +++ b/trans @@ -0,0 +1 @@ +Subproject commit 853ff7ab39325377685d306c2f562825dd6b0aeb