Parsing NMEA 0183 sentences in Python

My skipper has recently bought an NMEA wifi gateway, which means that the NMEA messages from the various onboard instruments on his yacht are broadcasted on the yacht’s wifi network. This makes it very easy to grab the NMEA messages, and start trying to make sense of them.

Below an example of parsing a handful of the message types, on a Garmin network (note: the data collection was done with the boat stationary, thus there’s no interesting info about speed, vmg etc in these graphs, that will have to wait until we are actually sailing… 😉

The first graph is a frequency plot over the various NMEA message types – the most frequent messages are (remember, the boat was stationary):

IIHDT – heading T
GPWCV – waypoint closure velocity
GPZDA – time & date
IIVWR – AWA AWS
IIVTG – track made good and ground speed
GPXTE – xross track error
IIVPW – speed parallel to wind
IIDBT – depth below transducer
GPGSV – satellites in view
IIHDM – magnetic heading
IIWHW – Speed thru water
GPGLL – lat & long
IIMWD – wind direction & speed
GPRMC – recommended minimum data

nmea_prefixes

The next plot shows the true wind speed and angle over a period of time.

nmea_TW_timeline

The last graph shows two polar plots over true and apparent wind.

nmea_wind

Finally, an idea for the main performance analysis screen:

nmea_timeline

import re
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import datetime as dt
import matplotlib.dates as pldt
from collections import Counter

def parse_time(sentence):
    pattern = r'[0-9]+-[0-9]+-[0-9]+ [0-9]+:[0-9]+:[0-9]+,[0-9]+'
    result = re.match(pattern,sentence)
    if result:
        result = result.group().split(' ')
        return result
    return None

### VMG ###
def parse_iivpw(sentence):
    pattern = r'\$IIVPW,[0-9]+\.[0-9]+,N'
    result = re.findall(pattern,sentence)
    if result:
        result = result[0]
        result = result.split(',')
        return result
    return None
        
### TWA, TWS ###
def parse_iivwt(sentence):
    pattern = r"\$IIVWT,[0-9]+,(?:L|R),[0-9]+\.[0-9]+,N,[0-9]+\.[0-9]+,M"
    result = re.findall(pattern,sentence)
    if result:
        result = result[0]
        result = result.split(',')
        return result
    return None

### AWA, AWS ###
def parse_iivwr(sentence):
    pattern = r"\$IIVWR,[0-9]+,(?:L|R),[0-9]+\.[0-9]+,N,[0-9]+\.[0-9]+,M"
    result = re.findall(pattern,sentence)
    if result:
        result = result[0]
        result = result.split(',')
        return result
    return None

### STW ###
def parse_iivhw(sentence):
    pattern = r'\$IIVHW,,T,,M,[0-9]+\.[0-9]+,N,[0-9]+\.[0-9]+,K'
    result = re.findall(pattern,sentence)
    if result:
        result = result[0]
        result = result.split(',')
        return result

def prefix(sentence):
    pattern = r'\$[A-Z]+,'
    result = re.findall(pattern,sentence)
    if result:
        result = result[0]
        result = result.split(',')
        
        return result[0]
    return None

def degtorad(twa,side):
    if side == 'L':
        twa = twa * -1
    return np.deg2rad(twa)

def strip_leading_zero(s):
    if s[0] == '0':
        return s[1:]

all_iivwr = []
all_iivwt = []
all_iivhw = []
all_iivpw = []
all_prefixes = []

f = open ('udp_rx.log','r')
nr_lines = 0
for line in f:
    already_parsed = False
    
    time = line.split()[1]
    pf = prefix(line)
    if pf != None:
        all_prefixes.append(pf)
    
    msg = parse_iivwr(line)
    if msg:
        msg.append(time)
        all_iivwr.append(msg)
        already_parsed = True
        
    if not already_parsed:    
        msg = parse_iivwt(line)
        if msg:
            msg.append(time)
            all_iivwt.append(msg)
            already_parsed = True
            
    if not already_parsed:        
        msg = parse_iivhw(line)
        if msg:
            msg.append(time)
            all_iivhw.append(msg)
            already_parsed = True
            
    if not already_parsed:        
        msg = parse_iivpw(line)
        if msg:
            msg.append(time)
            all_iivpw.append(msg)
            
    nr_lines += 1

prefix_counts = Counter(all_prefixes)

iivpw_df = pd.DataFrame(all_iivpw)
iivpw_df.columns = ['TYPE','VMG','N','TIME']
iivpw_df.drop('N',inplace=True,axis=1)
iivpw_df['TIME'] = iivpw_df['TIME'].astype(dt.datetime)
iivpw_df.set_index('TIME',inplace=True)

#remove empty elements in sublists
all_iivhw = [ [item for item in sublist if item] for sublist in all_iivhw]

iivhw_df = pd.DataFrame(all_iivhw)
iivhw_df.columns = ['TYPE','T','M','KNOTS','N','KM','K','TIME']
iivhw_df['KNOTS'] = pd.to_numeric(iivhw_df['KNOTS'])
iivhw_df['KM'] = pd.to_numeric(iivhw_df['KM'])
iivhw_df.set_index('TIME',inplace=True)
iivhw_df.drop(['T','M','N','KM','K'],inplace=True,axis=1)

iivwt_df = pd.DataFrame(all_iivwt)
iivwt_df.columns = ['TYPE','TWA','SIDE','KNOTS','N','MS','M','TIME']
iivwt_df['TWA'] = iivwt_df['TWA'].apply( strip_leading_zero)
iivwt_df['TWA'] = pd.to_numeric(iivwt_df['TWA'])
iivwt_df['KNOTS'] = iivwt_df['KNOTS'].apply( strip_leading_zero)
iivwt_df['KNOTS'] = pd.to_numeric(iivwt_df['KNOTS'])
iivwt_df['MS'] = iivwt_df['MS'].apply( strip_leading_zero)
iivwt_df['MS'] = pd.to_numeric(iivwt_df['MS'])
iivwt_df['RAD'] = np.vectorize(degtorad)(iivwt_df['TWA'],iivwt_df['SIDE'])
iivwt_df.set_index('TIME',inplace=True)
iivwt_df.drop(['KNOTS','N','M'],inplace=True,axis=1)

iivwr_df = pd.DataFrame(all_iivwr)
iivwr_df.columns = ['TYPE','TWA','SIDE','KNOTS','N','MS','M','TIME']
iivwr_df['TWA'] = iivwr_df['TWA'].apply( strip_leading_zero)
iivwr_df['TWA'] = pd.to_numeric(iivwr_df['TWA'])
iivwr_df['KNOTS'] = iivwr_df['KNOTS'].apply( strip_leading_zero)
iivwr_df['KNOTS'] = pd.to_numeric(iivwr_df['KNOTS'])
iivwr_df['MS'] = iivwr_df['MS'].apply( strip_leading_zero)
iivwr_df['MS'] = pd.to_numeric(iivwr_df['MS'])
iivwr_df['RAD'] = np.vectorize(degtorad)(iivwr_df['TWA'],iivwr_df['SIDE'])
iivwr_df.set_index('TIME',inplace=True)
iivwr_df.drop(['KNOTS','N','M'],inplace=True,axis=1)

print (iivhw_df.head())
print (iivwt_df.head())
print (iivwr_df.head())
print (iivpw_df.head())

print ('max TWS:',iivwt_df['MS'].max(), 'm/s')
print ('min TWS:',iivwt_df['MS'].min(), 'm/s')
print ('max TWA:',np.rad2deg(iivwt_df['RAD'].max()), 'deg')
print ('min TWA:',np.rad2deg(iivwt_df['RAD'].min()), 'deg')
print ('max AWS:',iivwr_df['MS'].max(), 'm/s')
print ('min AWS:',iivwr_df['MS'].min(), 'm/s') 
print ('max AWA:',np.rad2deg(iivwr_df['RAD'].max()), 'deg')
print ('mix AWA:',np.rad2deg(iivwr_df['RAD'].min()), 'deg')
print ('max STW:',iivhw_df['KNOTS'].max(),'knots')
print ('min STW:',iivhw_df['KNOTS'].min(),'knots')
print ('max VMG:',iivpw_df['VMG'].max(),'knots')
print ('min VMG:',iivpw_df['VMG'].min(),'knots')
       
joint = pd.concat([iivwt_df,iivhw_df,iivwr_df,iivpw_df],axis=0)
joint.sort_index(inplace=True)

joint = joint[['TYPE','KNOTS','MS','TWA','SIDE','VMG']]
cols = ['TYPE','BOAT_SPEED','WIND_SPEED','WIND_ANGLE','SIDE','VMG']
joint.columns = cols
joint.index = joint.index.astype(dt.datetime)

print (joint.head())

print ('number of true wind data:',len(iivwt_df))
print ('number of apparent wind data:',len(iivwr_df))
print ('number of STW data:',len(iivhw_df))
print ('number of VMG data:',len(iivpw_df))
print ('total number of sentences:',nr_lines - 3)

### PATTERN ###
#pfix_labels,pfix_values = zip(*prefix_counts.items())
###

foo = prefix_counts.most_common()
pfix_labels,pfix_values = zip(*foo)

plt.figure(figsize=(18,12))
ax = plt.subplot(121,polar=True)
ax.set_theta_zero_location("N")
ax.set_theta_direction(-1)
ax.set_title('TWA & TWS [m/s]')

tw_thetas = iivwt_df['RAD']
tw_radi = iivwt_df['MS']
aw_thetas = iivwr_df['RAD']
aw_radi = iivwr_df['MS']
tws_max = max(tw_radi)
aws_max = max(aw_radi)

max_radi = max(tws_max,aws_max)

ax.set_ylim(0,np.ceil(max_radi))
tw = ax.scatter(tw_thetas,tw_radi,marker='.',
                facecolors='none',c=tw_radi,cmap='jet',alpha=0.7)
tw_cbar = plt.colorbar(tw)
tw_cbar.set_label('TWS [m/s]')

ax = plt.subplot(122,polar=True)
ax.set_theta_zero_location("N")
ax.set_theta_direction(-1)
ax.set_title('AWA & AWS [m/s]')
ax.set_ylim(0,np.ceil(max_radi))

aw = ax.scatter(aw_thetas,aw_radi,marker='|',c=aw_radi,cmap='jet',alpha=0.7)
aw_cbar = plt.colorbar(aw)
aw_cbar.set_label('AWS [m/s]')

plt.tight_layout()
plt.savefig('nmea_wind_polar.jpg',format='jpg')

plt.figure(figsize=(18,12))
ax1 = plt.subplot(2,1,1)
plt.title('TWS')
every_nth = 100
plt.grid(which='major')
ax1.plot(iivwt_df.index,iivwt_df['MS'])
for n,label in enumerate(ax1.xaxis.get_ticklabels()):
    if n % every_nth !=0:
        label.set_visible(False)
ax1.set_ylabel('TWS [m/s]')

ax2 = plt.subplot(2,1,2,sharex=ax1)
plt.title('TWA')
plt.grid(which='major')
plt.plot(iivwt_df.index,np.rad2deg(iivwt_df['RAD']))
for n,label in enumerate(ax2.xaxis.get_ticklabels()):
    if n % every_nth !=0:
        label.set_visible(False)

ax2.set_ylabel('TWA [deg]')
plt.savefig('nmea_TW_timeline.jpg',format='jpg')

plt.figure(figsize=(18,12))
plt.title('NMEA message frequency on Garmin network')
plt.grid(which='both')
plt.ylabel('Message count')
plt.xlabel('Message prefix')

xticks = pfix_labels
plt.bar(range(len(pfix_values)),pfix_values)
plt.xticks(range(len(pfix_values)),xticks,rotation='vertical')
plt.savefig('nmea_prefixes.jpg',format='jpg')
plt.show()

 

About swdevperestroika

High tech industry veteran, avid hacker reluctantly transformed to mgmt consultant.
This entry was posted in Data Analytics, Maritime Technology, Nautical Information Systems, NMEA, Python and tagged , , , , , . Bookmark the permalink.

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out /  Change )

Google photo

You are commenting using your Google account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )

Connecting to %s