[FFmpeg-devel] [PATCH v2 4/5] avutil/channel_layout: add av_channel_layout_retype()

[Marton Balint]

v2: add conversion from custom layout to ambisonic

Signed-off-by: Marton Balint <cus at passwd.hu>
---
 doc/APIchanges             |  3 ++
 libavutil/channel_layout.c | 74 ++++++++++++++++++++++++++++++++++++++
 libavutil/channel_layout.h | 11 ++++++
 libavutil/version.h        |  2 +-
 4 files changed, 89 insertions(+), 1 deletion(-)

diff --git a/doc/APIchanges b/doc/APIchanges
index 8e8498f803..ce1e816fa5 100644
--- a/doc/APIchanges
+++ b/doc/APIchanges
@@ -2,6 +2,9 @@ The last version increases of all libraries were on 2023-02-09
 
 API changes, most recent first:
 
+2024-02-xx - xxxxxxxxxx - lavu 58.38.100 - channel_layout.h
+  Add av_channel_layout_retype().
+
 2024-02-xx - xxxxxxxxxx - lavu 58.37.100 - channel_layout.h
   Add av_channel_layout_from_custom().
 
diff --git a/libavutil/channel_layout.c b/libavutil/channel_layout.c
index 0810d32bf6..056b7f7c10 100644
--- a/libavutil/channel_layout.c
+++ b/libavutil/channel_layout.c
@@ -1036,3 +1036,77 @@ uint64_t av_channel_layout_subset(const AVChannelLayout *channel_layout,
 
     return ret;
 }
+
+static int64_t masked_description(AVChannelLayout *channel_layout, int start_channel)
+{
+    uint64_t mask = 0;
+    for (int i = start_channel; i < channel_layout->nb_channels; i++) {
+        enum AVChannel ch = channel_layout->u.map[i].id;
+        if (ch >= 0 && ch < 63 && mask < (1ULL << ch))
+            mask |= (1ULL << ch);
+        else
+            return AVERROR(EINVAL);
+    }
+    return mask;
+}
+
+int av_channel_layout_retype(AVChannelLayout *channel_layout, enum AVChannelOrder order)
+{
+    if (!av_channel_layout_check(channel_layout))
+        return AVERROR(EINVAL);
+
+    if (channel_layout->order == order)
+        return 0;
+
+    switch (order) {
+    case AV_CHANNEL_ORDER_UNSPEC: {
+        int nb_channels = channel_layout->nb_channels;
+        av_channel_layout_uninit(channel_layout);
+        channel_layout->order       = AV_CHANNEL_ORDER_UNSPEC;
+        channel_layout->nb_channels = nb_channels;
+        return 0;
+        }
+    case AV_CHANNEL_ORDER_CUSTOM: {
+        AVChannelLayout custom = { 0 };
+        int ret = av_channel_layout_from_custom(&custom, channel_layout->nb_channels);
+        if (ret < 0)
+            return ret;
+        if (channel_layout->order != AV_CHANNEL_ORDER_UNSPEC)
+            for (int i = 0; i < channel_layout->nb_channels; i++)
+                custom.u.map[i].id = av_channel_layout_channel_from_index(channel_layout, i);
+        av_channel_layout_uninit(channel_layout);
+        *channel_layout = custom;
+        return 0;
+        }
+    case AV_CHANNEL_ORDER_NATIVE:
+        if (channel_layout->order == AV_CHANNEL_ORDER_CUSTOM) {
+            int64_t mask = masked_description(channel_layout, 0);
+            if (mask < 0)
+                return 1;
+            av_channel_layout_uninit(channel_layout);
+            return av_channel_layout_from_mask(channel_layout, mask);
+        } else {
+            return 1;
+        }
+    case AV_CHANNEL_ORDER_AMBISONIC:
+        if (channel_layout->order == AV_CHANNEL_ORDER_CUSTOM) {
+            int64_t mask;
+            int nb_channels = channel_layout->nb_channels;
+            int order = ambisonic_order(channel_layout);
+            if (order < 0)
+                return 1;
+            mask = masked_description(channel_layout, (order + 1) * (order + 1));
+            if (mask < 0)
+                return 1;
+            av_channel_layout_uninit(channel_layout);
+            channel_layout->order       = AV_CHANNEL_ORDER_AMBISONIC;
+            channel_layout->nb_channels = nb_channels;
+            channel_layout->u.mask      = mask;
+            return 0;
+        } else {
+            return 1;
+        }
+    default:
+        return 1;
+    }
+}
diff --git a/libavutil/channel_layout.h b/libavutil/channel_layout.h
index 37629ab5d2..7e27a00d39 100644
--- a/libavutil/channel_layout.h
+++ b/libavutil/channel_layout.h
@@ -817,6 +817,17 @@ int av_channel_layout_check(const AVChannelLayout *channel_layout);
  */
 int av_channel_layout_compare(const AVChannelLayout *chl, const AVChannelLayout *chl1);
 
+/**
+ * Try changing the AVChannelOrder of a channel layout.
+ *
+ * @param channel_layout channel layout which will be changed
+ * @param order the desired channel layout order
+ * @return 0 on success or if the channel layout is already in the desired order
+ *         1 if using the desired order is not possible for the specified layout
+ *         AVERROR code on error
+ */
+int av_channel_layout_retype(AVChannelLayout *channel_layout, enum AVChannelOrder order);
+
 /**
  * @}
  */
diff --git a/libavutil/version.h b/libavutil/version.h
index 3b38f8f5da..cebf4a0acd 100644
--- a/libavutil/version.h
+++ b/libavutil/version.h
@@ -79,7 +79,7 @@
  */
 
 #define LIBAVUTIL_VERSION_MAJOR  58
-#define LIBAVUTIL_VERSION_MINOR  37
+#define LIBAVUTIL_VERSION_MINOR  38
 #define LIBAVUTIL_VERSION_MICRO 100
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \
-- 
2.35.3