Data in Motion - Precipitation Map

Read Time: 4 minutes

Today is again a lighter post playing with visualizations. The data focus is on the Standardized Precipitation Index data for the U.S. over that last one-hundred years. Static images and data can be useful, but visualizing data over time can be a welcome addition for analysis. So I’ll be converting data ultimately into a video of the data over time using primarily F#.

The above video starts with the data. In its raw form it is county-level precipitation data from 1895-2016. More specifically, the Standardized Precipitation Index (SPI) is a standard deviation metric, and for this dataset values range from -3 to 3. So this shows over time is how much areas deviate from the relative mean. The static data is brought together using a combination of tools. The primary one being F#, along with the libraries Deedle and Plotly.NET for data manipulation and chart creation (respectively). It is pulled together using ffmpeg in order to transform a series of images into a final video.

Source Data: Centers for Disease Control and Prevention. National Environmental Public Health Tracking Network. https://data.cdc.gov/Environmental-Health-Toxicology/Standardized-Precipitation-Index-1895-2016/xbk2-5i4e

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
open System
open System.Diagnostics
open System.IO
open Deedle
open Newtonsoft.Json
open Plotly.NET
open Plotly.NET.ImageExport

/// Convert a datafile into an imagefile name (with no extension)
let buildImageNameNoExtension i _dataFileName =
  Path.Combine("images", sprintf "image_%04d" i)

/// Convert a datafile name into a year-month chart title
let fileNameToTitle dataFileName =
  let regex = Text.RegularExpressions.Regex("precipitation_(\d+)_(\d+).csv")

  let matches = regex.Match(dataFileName)
  let year = matches.Groups.[1].Captures.[0].ToString() |> int
  let month = matches.Groups.[2].Captures.[0].ToString() |> int

  sprintf "%4d-%02d" year month

/// Json object of county code to map coordinates polygon
/// Source: https://raw.githubusercontent.com/plotly/datasets/master/geojson-counties-fips.json
let geoJson =
  IO.File.ReadAllText("data/geojson-counties-fips.json")
  |> JsonConvert.DeserializeObject

/// Build map of precipitation data
let buildMap index dataFile =
  let title = fileNameToTitle dataFile

  let data = Frame.ReadCsv(dataFile, true, separators = ",")

  let fips =
    data
    |> Frame.getCol "countyfips"
    |> Series.values
    |> Array.ofSeq

  let spi =
    data
    |> Frame.getCol "spi"
    |> Series.values
    |> Array.ofSeq

  let chart =
    Chart.ChoroplethMap (
      locations = fips,
      z = spi,
      Locationmode = StyleParam.LocationFormat.GeoJson_Id,
      GeoJson = geoJson,
      FeatureIdKey = "id",
      Colorscale = 
        StyleParam.Colorscale.Custom([
          (0.0,   "#3d0c06")
          (0.165, "#6d0c06")
          (0.25,  "#9d0c06")
          (0.5 ,  "#9a9a9a")
          (0.665, "#060c9d")
          (0.75,  "#060c6d")
          (1.0,   "#060c3d")
          ]),
      Zmin = -3,
      Zmax = 3)
    |> Chart.withMap (Geo.init (Scope = StyleParam.GeoScope.Usa))
    |> Chart.withColorBarStyle ("SPI", Length = 0.75)
    |> Chart.withTitle (title=title, Titlefont=Font.init(Family=StyleParam.FontFamily.Courier_New, Size=32.))
    |> Chart.withSize (800., 500.)
    |> Chart.savePNG (path = buildImageNameNoExtension index dataFile, Width = 800, Height = 500)

/// Execute command
let exec command args =
  let startInfo = ProcessStartInfo(FileName = command, Arguments = args)
  let p = new Process(StartInfo = startInfo)

  let success = p.Start()
  if not success then
    printfn "Process Failed"
  else 
    p.WaitForExit()

/// Build a video (mp4) using all pngs in the sourceDir
let buildVideo sourceDir dstFile =
  exec "ffmpeg" $"-y -i {sourceDir}/image_%%04d.png -c:v libx264 -vf fps=5 -pix_fmt yuv420p {dstFile}"

/// Convert an mp4 to a different file format (i.e. webm or .gif)
let convertVideo (inputFile: string) (outputFile: string) =
  exec "ffmpeg" $"-i {inputFile} {outputFile}"

[<EntryPoint>]
let main argv =
  // Create map images for each month of the data series
  // Name the images numerically, for consumption by ffmpeg  
  IO.Directory.GetFiles("./data", "precipitation*.csv")
  |> Array.sort
  |> Array.mapi (fun i x -> (i, x))
  |> Array.iter (fun (i, x) -> buildMap i x)

  // Combine images into a video
  buildVideo "images" "precipitation.mp4" |> ignore
  convertVideo "precipitation.mp4" "precipitation.webm" |> ignore

  0
Related Posts: