added extracting & submitting web forms tutorial

Author: x4nth055

README.md

@@ -76,6 +76,7 @@ This is a repository of all the tutorials of [The Python Code](https://www.thepy
     - [How to Convert HTML Tables into CSV Files in Python](https://www.thepythoncode.com/article/convert-html-tables-into-csv-files-in-python). ([code](web-scraping/html-table-extractor))
     - [How to Use Proxies to Anonymize your Browsing and Scraping using Python](https://www.thepythoncode.com/article/using-proxies-using-requests-in-python). ([code](web-scraping/using-proxies))
     - [How to Extract Script and CSS Files from Web Pages in Python](https://www.thepythoncode.com/article/extract-web-page-script-and-css-files-in-python). ([code](web-scraping/webpage-js-css-extractor))
+    - [How to Extract and Submit Web Forms from a URL using Python](https://www.thepythoncode.com/article/extracting-and-submitting-web-page-forms-in-python). ([code](web-scraping/extract-and-fill-forms))
 
 - ### [Python Standard Library](https://www.thepythoncode.com/topic/python-standard-library)
     - [How to Transfer Files in the Network using Sockets in Python](https://www.thepythoncode.com/article/send-receive-files-using-sockets-python). ([code](general/transfer-files/))

web-scraping/extract-and-fill-forms/README.md

@@ -0,0 +1,12 @@
+# [How to Extract and Submit Web Forms from a URL using Python](https://www.thepythoncode.com/article/extracting-and-submitting-web-page-forms-in-python)
+To run this:
+- `pip3 install -r requirements.txt`
+- To extract forms, use `form_extractor.py`:
+    ```
+    python form_extractor.py https://wikipedia.org
+    ```
+- To extract and submit forms, use `form_submitter.py`:
+    ```
+    python form_submitter.py https://wikipedia.org
+    ```
+    This will extract the first form (you can change that in the code) and prompt the user for each non-hidden input field, and then submits the form and loads the respond HTML in your default web browser, try it out!

web-scraping/extract-and-fill-forms/form_extractor.py

@@ -0,0 +1,56 @@
+from bs4 import BeautifulSoup
+from requests_html import HTMLSession
+from pprint import pprint
+
+# initialize an HTTP session
+session = HTMLSession()
+
+
+def get_all_forms(url):
+    """Returns all form tags found on a web page's `url` """
+    # GET request
+    res = session.get(url)
+    # for javascript driven website
+    # res.html.render()
+    soup = BeautifulSoup(res.html.html, "html.parser")
+    return soup.find_all("form")
+
+
+def get_form_details(form):
+    """Returns the HTML details of a form,
+    including action, method and list of form controls (inputs, etc)"""
+    details = {}
+    # get the form action (requested URL)
+    action = form.attrs.get("action").lower()
+    # get the form method (POST, GET, DELETE, etc)
+    # if not specified, GET is the default in HTML
+    method = form.attrs.get("method", "get").lower()
+    # get all form inputs
+    inputs = []
+    for input_tag in form.find_all("input"):
+        # get type of input form control
+        input_type = input_tag.attrs.get("type", "text")
+        # get name attribute
+        input_name = input_tag.attrs.get("name")
+        # get the default value of that input tag
+        input_value =input_tag.attrs.get("value", "")
+        # add everything to that list
+        inputs.append({"type": input_type, "name": input_name, "value": input_value})
+    # put everything to the resulting dictionary
+    details["action"] = action
+    details["method"] = method
+    details["inputs"] = inputs
+    return details
+
+
+if __name__ == "__main__":
+    import sys
+    # get URL from the command line
+    url = sys.argv[1]
+    # get all form tags
+    forms = get_all_forms(url)
+    # iteratte over forms
+    for i, form in enumerate(forms, start=1):
+        form_details = get_form_details(form)
+        print("="*50, f"form #{i}", "="*50)
+        pprint(form_details)

web-scraping/extract-and-fill-forms/form_submitter.py

@@ -0,0 +1,63 @@
+from bs4 import BeautifulSoup
+from requests_html import HTMLSession
+
+from pprint import pprint
+from urllib.parse import urljoin
+import webbrowser
+import sys
+
+from form_extractor import get_all_forms, get_form_details, session
+
+# get the URL from the command line
+url = sys.argv[1]
+# get the first form (edit this as you wish)
+first_form = get_all_forms(url)[0]
+# extract all form details
+form_details = get_form_details(first_form)
+pprint(form_details)
+# the data body we want to submit
+data = {}
+for input_tag in form_details["inputs"]:
+    if input_tag["type"] == "hidden":
+        # if it's hidden, use the default value
+        data[input_tag["name"]] = input_tag["value"]
+    elif input_tag["type"] != "submit":
+        # all others except submit, prompt the user to set it
+        value = input(f"Enter the value of the field '{input_tag['name']}' (type: {input_tag['type']}): ")
+        data[input_tag["name"]] = value
+
+# join the url with the action (form request URL)
+url = urljoin(url, form_details["action"])
+
+if form_details["method"] == "post":
+    res = session.post(url, data=data)
+elif form_details["method"] == "get":
+    res = session.get(url, params=data)
+
+# the below code is only for replacing relative URLs to absolute ones
+soup = BeautifulSoup(res.content, "html.parser")
+for link in soup.find_all("link"):
+    try:
+        link.attrs["href"] = urljoin(url, link.attrs["href"])
+    except:
+        pass
+for script in soup.find_all("script"):
+    try:
+        script.attrs["src"] = urljoin(url, script.attrs["src"])
+    except:
+        pass
+for img in soup.find_all("img"):
+    try:
+        img.attrs["src"] = urljoin(url, img.attrs["src"])
+    except:
+        pass
+for a in soup.find_all("a"):
+    try:
+        a.attrs["href"] = urljoin(url, a.attrs["href"])
+    except:
+        pass
+
+# write the page content to a file
+open("page.html", "w").write(str(soup))
+# open the page on the default browser
+webbrowser.open("page.html")    

web-scraping/extract-and-fill-forms/requirements.txt

@@ -0,0 +1,2 @@
+requests_html
+bs4