Python urllib

Built-in HTTP client library with request/response handling examples.

HTTP Methods

GET Request

import urllib.request
import urllib.parse
import json

# Simple GET request
url = "https://api.github.com/users/octocat"
response = urllib.request.urlopen(url)
data = response.read()
print(data.decode('utf-8'))

# GET with parameters
params = {'q': 'python', 'sort': 'stars'}
url = "https://api.github.com/search/repositories?" + urllib.parse.urlencode(params)
response = urllib.request.urlopen(url)
data = json.loads(response.read().decode('utf-8'))
print(data)

POST Request

import urllib.request
import urllib.parse
import json

# POST with JSON data
url = "https://httpbin.org/post"
data = {"name": "John", "age": 30}
json_data = json.dumps(data).encode('utf-8')

req = urllib.request.Request(url, data=json_data)
req.add_header('Content-Type', 'application/json')
response = urllib.request.urlopen(req)
result = response.read().decode('utf-8')
print(result)

# POST with form data
form_data = urllib.parse.urlencode({'username': 'john', 'password': 'secret'}).encode('utf-8')
req = urllib.request.Request(url, data=form_data)
response = urllib.request.urlopen(req)
print(response.read().decode('utf-8'))

PUT Request

import urllib.request
import urllib.parse
import json

# PUT request to update resource
url = "https://httpbin.org/put"
data = {"id": 1, "name": "Updated Name", "status": "active"}
json_data = json.dumps(data).encode('utf-8')

req = urllib.request.Request(url, data=json_data, method='PUT')
req.add_header('Content-Type', 'application/json')
response = urllib.request.urlopen(req)
result = response.read().decode('utf-8')
print(result)

# PUT with custom headers
headers = {
    'Authorization': 'Bearer your-token-here',
    'Content-Type': 'application/json'
}
req = urllib.request.Request(url, data=json_data, method='PUT', headers=headers)
response = urllib.request.urlopen(req)
print(response.read().decode('utf-8'))

DELETE Request

import urllib.request
import urllib.parse

# DELETE request
url = "https://httpbin.org/delete"
req = urllib.request.Request(url, method='DELETE')
response = urllib.request.urlopen(req)
result = response.read().decode('utf-8')
print(result)

# DELETE with authentication
headers = {
    'Authorization': 'Bearer your-token-here'
}
req = urllib.request.Request(url, method='DELETE', headers=headers)
response = urllib.request.urlopen(req)
print(response.read().decode('utf-8'))

# DELETE with query parameters
params = {'id': '123'}
url_with_params = url + '?' + urllib.parse.urlencode(params)
req = urllib.request.Request(url_with_params, method='DELETE')
response = urllib.request.urlopen(req)
print(response.read().decode('utf-8'))

Advanced Features

Error Handling

import urllib.request
import urllib.error

def make_request(url):
    try:
        response = urllib.request.urlopen(url)
        data = response.read().decode('utf-8')
        return data
    except urllib.error.HTTPError as e:
        print(f"HTTP Error {e.code}: {e.reason}")
        return None
    except urllib.error.URLError as e:
        print(f"URL Error: {e.reason}")
        return None
    except Exception as e:
        print(f"Unexpected error: {e}")
        return None

# Usage
url = "https://httpbin.org/status/404"
result = make_request(url)
if result:
    print(result)
else:
    print("Request failed")

File Download

import urllib.request
import os

def download_file(url, filename):
    try:
        # Download file
        urllib.request.urlretrieve(url, filename)
        print(f"File downloaded: {filename}")
        
        # Get file info
        file_size = os.path.getsize(filename)
        print(f"File size: {file_size} bytes")
        
    except Exception as e:
        print(f"Download failed: {e}")

# Download a file
url = "https://httpbin.org/robots.txt"
filename = "robots.txt"
download_file(url, filename)

# Download with progress
def download_with_progress(url, filename):
    def progress_hook(block_num, block_size, total_size):
        downloaded = block_num * block_size
        if total_size > 0:
            percent = min(100, (downloaded * 100) // total_size)
            print(f"\rDownloaded: {percent}%", end='')
    
    urllib.request.urlretrieve(url, filename, progress_hook)
    print(f"\nDownload complete: {filename}")

# download_with_progress(url, filename)

Quick Reference

Common Modules:

• urllib.request - HTTP requests
• urllib.parse - URL parsing
• urllib.error - Exception handling
• urllib.robotparser - robots.txt parsing

Key Functions:

• urlopen() - Open URLs
• urlretrieve() - Download files
• urlencode() - Encode parameters
• Request() - Create requests