Beautiful Soup implementation
import pandas as pd import requests from bs4 import BeautifulSoup import random import mysql.connector #url = "http://somewebsite?page=1" # Create a list in a range of 10-20 mydb = mysql.connector.connect( host="localhost", user="root", password="password", database="dbname" ) my_list = [*range(1, 4, 1)] for iurl in my_list: # ("Enabling special output mode (% s)") % (currentValue) url = "http://somewebsite?page={}".format(iurl) # Create a list in a range of 10-20 #my_list = [*range(1, 300, 1)] # Print the list # print(url) user_agents = [ "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0", "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0", "Mozilla/5.0 (X11; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0" ] random_user_agent = random.choice(user_agents) headers = { 'User-A