PROWAREtech

.NET: Download any Type of File Data from a URL with HttpClient

This article is compatible with .NET Core 3.1, .NET 5, .NET 6 and .NET 8. This code works equally as well with an ASP.NET Core web server application.

The following code will download any type of data from an Internet URL, decompress it if compressed and then write it to the console. See this article for an example of how to read data, including binary data, from a console application such as this one.

See also: Download using WebClient for a simplier client than HttpClient; GET Request from an Endpoint and POST Request to an Endpoint for using HttpClient with REST API endpoints.

using System;
using System.IO.Compression;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

namespace ConsoleApp1
{
	class Program
	{
		// NOTE: must specify that Main() be an async method
		static async Task Main(string[] args)
		{
			if(args.Length == 0)
			{
				Console.WriteLine("Must specify a URL to download");
				return;
			}

			using(var httpClient = new System.Net.Http.HttpClient())
			{

				// NOTE: to save bandwidth, request compressed content
				httpClient.DefaultRequestHeaders.AcceptEncoding.Clear();
				httpClient.DefaultRequestHeaders.AcceptEncoding.Add(new System.Net.Http.Headers.StringWithQualityHeaderValue("gzip"));
				httpClient.DefaultRequestHeaders.AcceptEncoding.Add(new System.Net.Http.Headers.StringWithQualityHeaderValue("deflate"));
				httpClient.DefaultRequestHeaders.AcceptEncoding.Add(new System.Net.Http.Headers.StringWithQualityHeaderValue("br"));

				// NOTE: accept all languages
				httpClient.DefaultRequestHeaders.AcceptLanguage.Clear();
				httpClient.DefaultRequestHeaders.AcceptLanguage.Add(new System.Net.Http.Headers.StringWithQualityHeaderValue("*"));

				// NOTE: accept all media types
				httpClient.DefaultRequestHeaders.Accept.Clear();
				httpClient.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("*/*"));

				System.Net.Http.HttpResponseMessage response;
				try
				{
					// NOTE: argument zero may be a URL to text, like HTML, or a zip file, image file, PDF file or any other binary data
					response = await httpClient.GetAsync(args[0]);
					using (var ms = new MemoryStream())
					{
						var contentType = response.Content.Headers.ContentType.ToString().ToLower();

						// NOTE: this block of code may be omitted; it simply writes the URL along with the content-type
						if (contentType.StartsWith("text/"))
						{
							var line = args[0] + " => " + contentType;
							Console.WriteLine(line);
							Console.WriteLine(new string('=', line.Length));
						}

						await response.Content.CopyToAsync(ms);
						byte[] data;
						switch (response.Content.Headers.ContentEncoding.FirstOrDefault())
						{
							case "br":
								data = await Decompressor.Brotli(ms);
								break;
							case "gzip":
								data = await Decompressor.GZip(ms);
								break;
							case "deflate":
								data = await Decompressor.Deflate(ms);
								break;
							default: // NOTE: not compressed
								data = ms.ToArray();
								break;
						}
						using (var msData = new MemoryStream(data))
						{
							msData.WriteTo(Console.OpenStandardOutput());
						}
					}
				}
				catch (Exception ex)
				{
					Console.WriteLine(ex.Message);
					return;
				}
			}
		}
	}
	// NOTE: this class is used to decompress data that is sent compressed, saving bandwidth
	internal class Decompressor
	{
		public static async Task<byte[]> GZip(MemoryStream inputStream, CancellationToken cancel = default)
		{
			inputStream.Position = 0;
			using (var outputStream = new MemoryStream())
			{
				using (var compressionStream = new GZipStream(inputStream, CompressionMode.Decompress))
				{
					await compressionStream.CopyToAsync(outputStream, cancel);
				}
				return outputStream.ToArray();
			}
		}
		public static async Task<byte[]> Brotli(MemoryStream inputStream, CancellationToken cancel = default)
		{
			inputStream.Position = 0;
			using (var outputStream = new MemoryStream())
			{
				using (var compressionStream = new BrotliStream(inputStream, CompressionMode.Decompress))
				{
					await compressionStream.CopyToAsync(outputStream, cancel);
				}
				return outputStream.ToArray();
			}
		}
		public static async Task<byte[]> Deflate(MemoryStream inputStream, CancellationToken cancel = default)
		{
			inputStream.Position = 0;
			using (var outputStream = new MemoryStream())
			{
				using (var compressionStream = new DeflateStream(inputStream, CompressionMode.Decompress))
				{
					await compressionStream.CopyToAsync(outputStream, cancel);
				}
				return outputStream.ToArray();
			}
		}
	}
}

Comment