deepcoder · September 3, 2025 10:49 · Jun 23, 2023 · Jun 19, 2023 · Jun 18, 2023 · Jun 18, 2023
diff --git a/gistfile1.txt b/gistfile1.txt
@@ -24,12 +24,7 @@ sudo vi /etc/systemd/system/birdnet_mqtt_py.service
 # content for this file :
 [Unit]
 Description=BirdNET MQTT Publish
-# does not seems to be a reason to keep shutting it down and relaunching with the other birdnet service
-# so it should stay running from boot now and watching the syslog
-#After=birdnet_analysis.service
-#Requires=birdnet_analysis.service
 [Service]
-RuntimeMaxSec=3600
 Restart=always
 Type=simple
 RestartSec=5

diff --git a/gistfile1.txt b/gistfile1.txt
@@ -24,8 +24,10 @@ sudo vi /etc/systemd/system/birdnet_mqtt_py.service
 # content for this file :
 [Unit]
 Description=BirdNET MQTT Publish
-After=birdnet_analysis.service
-Requires=birdnet_analysis.service
+# does not seems to be a reason to keep shutting it down and relaunching with the other birdnet service
+# so it should stay running from boot now and watching the syslog
+#After=birdnet_analysis.service
+#Requires=birdnet_analysis.service
 [Service]
 RuntimeMaxSec=3600
 Restart=always

diff --git a/gistfile1.txt b/gistfile1.txt
@@ -96,3 +96,10 @@ example lovelace
         icon: mdi:bird
 
 ```
+
+JSON that is generated:
+
+```
+{"ts": "1687047081.0", "sciname": "Baeolophus inornatus", "comname": "Oak Titmouse", "confidence": "0.7201002", "url": "http://en.wikipedia.org/wiki/Baeolophus_inornatus"}
+
+```
diff --git a/birdnet_to_mqtt.py b/birdnet_to_mqtt.py
@@ -0,0 +1,150 @@
+#! /usr/bin/env python3
+# birdnet_to_mqtt.py
+#
+# 202306171542
+#
+# monitor the records in the syslog file for info from the birdnet system on birds that it detects
+# publish this data to mqtt
+#
+
+import time
+import re
+import dateparser    
+import datetime
+import json
+import paho.mqtt.client as mqtt
+
+
+# this generator function monitors the requested file handle for new lines added at its end
+# the newly added line is returned by the function
+def file_row_generator(s):
+    s.seek(0,2)
+    while True :
+        line = s.readline()
+        if not line:
+            time.sleep(0.1)
+            continue
+        yield line
+
+# flag to select whether to process all detections, if False, only the records above the set threshold will be processed
+
+process_all = True
+
+# mqtt server
+mqtt_server = "192.168.2.242"
+
+# mqtt topic where all heard birds will be published
+mqtt_topic_all_birds = 'birdpi/all'
+
+# mqtt topic for bird heard above threshold will be published
+mqtt_topic_confident_birds = 'birdpi/confident'
+
+# url base for website that will be used to look up info about bird
+bird_lookup_url_base = 'http://en.wikipedia.org/wiki/'
+
+# regular expression patters used to decode the records from birdnet
+
+re_all_found = re.compile(r'birdnet_analysis\.sh.*\(.*\)')
+re_found_bird = re.compile(r'\(([^)]+)\)')
+re_log_timestamp = re.compile(r'.+?(?= birdnet-)')
+
+re_high_found = re.compile(r'(?<=python3\[).*\.mp3$')
+re_high_clean = re.compile(r'(?<=\]:).*\.mp3$')
+
+syslog = open('/var/log/syslog')
+
+# this little hack is to make each received record for the all birds section unique
+# the date and time that the log returns is only down to the 1 second accuracy, do
+# you can get multiple records with same date and time, this will make Home Assistant not
+# think there is a new reading so we add a incrementing tenth of second to each record received
+ts_noise = 0.0
+
+#try :
+# connect to MQTT server
+mqttc = mqtt.Client('birdnet_mqtt')  # Create instance of client with client ID
+mqttc.connect(mqtt_server, 1883)  # Connect to (broker, port, keepalive-time)
+mqttc.loop_start()
+
+# call the generator function and process each line that is returned
+for row in file_row_generator(syslog):
+    # if line in log is from 'birdnet_analysis.sh' routine, then operate on it
+
+    # if selected the process the line return for every detection, even below threshold, this generates a lot more records to MQTT
+    if process_all and re_all_found.search(row) :
+
+        # get time stamp of the log entry
+        timestamp = str(datetime.datetime.timestamp(dateparser.parse(re.search(re_log_timestamp, row).group(0))) + ts_noise)
+
+        ts_noise = ts_noise + 0.1
+        if ts_noise > 0.9 :
+            ts_noise = 0.0
+
+        # extract the scientific name, common name and confidence level from the log entry
+        res = re.search(re_found_bird, row).group(1).split(',', 1)
+
+        # messy code to deal with single and/or double quotes around scientific name and common name
+        # while keeping a single quote in string of common name if that is part of bird name
+        if '"' in res[0] :
+            res[0] = res[0].replace('"', '')
+        else :
+            res[0] = res[0].replace("'", "")
+
+        # scientific name of bird is found prior to the underscore character
+        # common name of bird is after underscore string
+        # remainder of string is the confidence level
+        sci_name = res[0].split('_', 1)[0]
+        com_name = res[0].split('_', 1)[1]
+        confid = res[1].replace(' ', '')
+
+        # build python structure of fields that we will then turn into a json string
+        bird = {}
+        bird['ts'] = timestamp
+        bird['sciname'] = sci_name
+        bird['comname'] = com_name
+        bird['confidence'] = confid
+        # build a url from scientific name of bird that can be used to lookup info about bird
+        bird['url'] = bird_lookup_url_base + sci_name.replace(' ', '_')
+
+        # convert to json string we can sent to mqtt
+        json_bird = json.dumps(bird)
+
+        print(json_bird)
+
+        mqttc.publish(mqtt_topic_all_birds, json_bird, 1)
+
+    # bird found above confidence level found, process it
+    if re_high_found.search(row) :
+
+        # this slacker regular expression work, extracts the data about the bird found from the log line
+        # I do the parse in two passes, because I did not know the re to do it in one!
+
+        raw_high_bird = re.search(re_high_found, row)
+        raw_high_bird = raw_high_bird.group(0)
+        raw_high_bird = re.search(re_high_clean, raw_high_bird)
+        raw_high_bird = raw_high_bird.group(0)
+
+        # the fields we want are separated by semicolons, so split
+        high_bird_fields = raw_high_bird.split(';')
+
+        # build a structure in python that will be converted to json
+        bird = {}
+
+        # human time in this record is in two fields, date and time. They are human format
+        # combine them together separated by a space and they turn the human data into a python
+        # timestamp
+        raw_ts = high_bird_fields[0] + ' ' + high_bird_fields[1]
+
+        bird['ts'] = str(datetime.datetime.timestamp(dateparser.parse(raw_ts)))
+        bird['sciname'] = high_bird_fields[2]
+        bird['comname'] = high_bird_fields[3]
+        bird['confidence'] = high_bird_fields[4]
+        # build a url from scientific name of bird that can be used to lookup info about bird
+        bird['url'] = bird_lookup_url_base + high_bird_fields[2].replace(' ', '_')
+
+        # convert to json string we can sent to mqtt
+        json_bird = json.dumps(bird)
+
+        print(json_bird)
+
+        mqttc.publish(mqtt_topic_confident_birds, json_bird, 1)
+
diff --git a/gistfile1.txt b/gistfile1.txt
@@ -0,0 +1,98 @@
+Here is a slight hack python3 program that monitors the syslog output from the BirdNET-Pi bird sound identifications and publishes these records to MQTT. This can be used in Home Assistant to create sensors for the bird identifications.
+
+The program can publish both the output of all birds that BirdNet thinks it hears and also just those above the confidence level that you have set in BirdNET-Pi.
+
+You will have to edit the attached python program 'birdnet_to_mqtt.py' to change the MQTT server, MQTT topics and whether to publish all or just the birds heard above the confidence level.
+
+NOTE: there are different python 'environments' I am not using the birdnet python environment, so any python libraries that are used here are install as root and are used by the python3 binary at: /usr/bin/python3
+
+You will need a couple non-standard python libraries:
+```
+sudo pip3 install dateparser
+sudo pip3 install paho-mqtt
+```
+copy the python program to /usr/local/bin
+
+```
+sudo cp birdnet_to_mqtt.py /usr/local/bin
+
+```
+setup a service to run the python program at startup
+
+```
+sudo vi /etc/systemd/system/birdnet_mqtt_py.service
+# content for this file :
+[Unit]
+Description=BirdNET MQTT Publish
+After=birdnet_analysis.service
+Requires=birdnet_analysis.service
+[Service]
+RuntimeMaxSec=3600
+Restart=always
+Type=simple
+RestartSec=5
+User=root
+ExecStart=/usr/bin/python3 /usr/local/bin/birdnet_to_mqtt.py
+[Install]
+WantedBy=multi-user.target
+
+```
+
+enable the python program as a service
+
+```
+sudo systemctl enable birdnet_mqtt_py.service
+sudo systemctl start birdnet_mqtt_py.service
+sudo systemctl status birdnet_mqtt_py.service
+```
+
+MQTT sensors in Home Assistant:
+
+```
+# birdpi
+  - name: "BirdPiNET Bird Heard 01"
+    unique_id: "BirdPiNet-01-heard"
+    state_topic: "birdpi/confident"
+    value_template: >
+      {% if value_json.ts is defined %}
+        "{{ value_json.ts }}"
+      {% else %}
+        "NA"
+      {% endif %}
+    json_attributes_topic: "birdpi/confident"
+
+  - name: "BirdPiNET Bird All 01"
+    unique_id: "BirdPiNet-01-all"
+    state_topic: "birdpi/all"
+    value_template: >
+      {% if value_json.ts is defined %}
+        "{{ value_json.ts }}"
+      {% else %}
+        "NA"
+      {% endif %}
+    json_attributes_topic: "birdpi/all"
+
+```
+
+example lovelace 
+
+```
+# https://github.com/gadgetchnnel/lovelace-home-feed-card
+    - type: custom:home-feed-card
+      title: BirdPiNET-Confident-01
+      show_empty: true
+      more_info_on_tap: true
+      scrollbars_enabled: false
+      max_item_count: 10
+      compact_mode: true
+      id_filter: ^home_feed_.*
+      entities:
+      - entity: sensor.birdpinet_bird_heard_01
+        more_info_on_tap: true
+        include_history: true
+        remove_repeats: false
+        max_history: 10
+        content_template: '{{comname}} {{sciname}} {{confidence}} {{ts}}'
+        icon: mdi:bird
+
+```