wellton/rosettadrone
1
0
Fork 0
Código Issues Pull Requests Ações Pacotes Projetos Versões Wiki Atividade

cleaning code and fix codec for WebRTC video streaming

Ver Código-Fonte
Esse commit está contido em:
Angel Ayala
2024-08-13 14:21:48 -03:00
pai a9a397a2d3
commit 02d4eb2587
7 arquivos alterados com 97 adições e 223 exclusões
Baixar Arquivo de Patch Baixar Arquivo de Diff Expandir todos os arquivos Colapsar todos os arquivos
app/src/main/java/sq/rogue/rosettadrone/plugins/WebRTC/DJIStreamer.java
+14 -39
Ver Arquivo
@@ -1,21 +1,12 @@
package sq.rogue.rosettadrone.plugins.WebRTC;
import android.content.Context;
//import android.os.Handler;
import android.util.Log;
import org.json.JSONException;
import org.json.JSONObject;
import org.webrtc.VideoCapturer;
import java.util.Hashtable;
import dji.common.product.Model;
//import dji.sdk.sdkmanager.DJISDKManager;
//import static io.socket.client.Socket.EVENT_DISCONNECT;
//import com.example.SocketConnection;
import sq.rogue.rosettadrone.plugins.WebRTC.websocket.Socket;
/**
@@ -23,16 +14,13 @@ import sq.rogue.rosettadrone.plugins.WebRTC.websocket.Socket;
* with clients, who desire videofeed.
*/
public class DJIStreamer {
private static final String TAG = "DJIStreamer";
private static final String TAG = DJIStreamer.class.getSimpleName();
// private String droneDisplayName = "";
private final Context context;
private final Hashtable<String, WebRTCClient> ongoingConnections = new Hashtable<>();
// private final SocketConnection socket;
private final Model aircraftModel;
public DJIStreamer(Context context, Model aircraftModel){
// this.droneDisplayName = DJISDKManager.getInstance().getProduct().getModel().getDisplayName();
this.aircraftModel = aircraftModel;
this.context = context;
setupSocketEvent();
@@ -63,35 +51,22 @@ public class DJIStreamer {
private void setupSocketEvent(){
Socket.getInstance().with(context).setOnEventResponseListener("webrtc_msg", (event, data) -> {
try {
JSONObject jsonData = new JSONObject(data);
String peerSocketID = jsonData.getString("socketID"); // The web-client sending a message
// Handler mainHandler = new Handler(context.getMainLooper());
// Runnable myRunnable = new Runnable() {
// @Override
// public void run() {
try {
Log.d(TAG, "Received WebRTCMessage data: " + data);
JSONObject jsonData = new JSONObject(data);
String peerSocketID = jsonData.getString("socketID"); // The web-client sending a message
Log.d(TAG, "Received WebRTCMessage: " + peerSocketID);
WebRTCClient client = getClient(peerSocketID);
WebRTCClient client = getClient(peerSocketID);
if (client == null){
// A new client wants to establish a P2P
client = addNewClient(peerSocketID);
}
if (client == null){
// A new client wants to establish a P2P
client = addNewClient(peerSocketID);
Log.d(TAG, "New WebRTCClient created");
}
// Then just pass the message to the client
// JSONObject message = (JSONObject) args[1];
client.handleWebRTCMessage(jsonData);
} catch (JSONException e) {
Log.e(TAG, "ERROR: Receiving WebRTCMessage: " + e.getMessage());
throw new RuntimeException(e);
}
// }
// };
// mainHandler.post(myRunnable);
// Then just pass the message to the client
client.handleWebRTCMessage(jsonData);
} catch (JSONException e) {
Log.e(TAG, "JSONException: " + e.getMessage());
}
});
}
}
app/src/main/java/sq/rogue/rosettadrone/plugins/WebRTC/DJIVideoCapturer.java
+53 -17
Ver Arquivo
@@ -5,6 +5,7 @@
package sq.rogue.rosettadrone.plugins.WebRTC;
import org.webrtc.CapturerObserver;
import org.webrtc.JavaI420Buffer;
import org.webrtc.NV12Buffer;
import org.webrtc.SurfaceTextureHelper;
import org.webrtc.VideoCapturer;
@@ -12,6 +13,7 @@ import org.webrtc.VideoFrame;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.os.SystemClock;
import android.util.Log;
@@ -25,7 +27,7 @@ import dji.sdk.camera.VideoFeeder;
import dji.sdk.codec.DJICodecManager;
public class DJIVideoCapturer implements VideoCapturer {
private final static String TAG = "DJIStreamer";
private final static String TAG = DJIVideoCapturer.class.getSimpleName();
private static DJICodecManager codecManager;
private static final ArrayList<CapturerObserver> observers = new ArrayList<CapturerObserver>();
@@ -39,7 +41,6 @@ public class DJIVideoCapturer implements VideoCapturer {
}
private void setupVideoListener(){
Log.d(TAG, "setupVideoListener");
if(codecManager != null) {
Log.d(TAG, "codecManager not null");
return;
@@ -53,34 +54,69 @@ public class DJIVideoCapturer implements VideoCapturer {
public void onYuvDataReceived(MediaFormat mediaFormat, ByteBuffer videoBuffer, int dataSize, int width, int height) {
if (videoBuffer != null){
try{
// TODO: We need to check which color format they are using by doing a lookup in our MediaFormat, otherwise we get green artifacts
// This can change with Android/device versions. The format might actually change, seemingly at random, according to community reports...
// Other possible buffers we might have to use: I420Buffer
long timestampNS = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime());
NV12Buffer buffer = new NV12Buffer(width,
height,
mediaFormat.getInteger(MediaFormat.KEY_STRIDE),
mediaFormat.getInteger(MediaFormat.KEY_SLICE_HEIGHT),
videoBuffer,
null);
VideoFrame videoFrame = new VideoFrame(buffer, 0, timestampNS);
VideoFrame videoFrame;
// Check the color format. Could create more cases if needed
int colorFormat = mediaFormat.getInteger(MediaFormat.KEY_COLOR_FORMAT);
switch (colorFormat) {
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible:
// NV12 Buffer
NV12Buffer nv12Buffer = new NV12Buffer(width, height,
mediaFormat.getInteger(MediaFormat.KEY_STRIDE),
mediaFormat.getInteger(MediaFormat.KEY_SLICE_HEIGHT),
videoBuffer, null);
videoFrame = new VideoFrame(nv12Buffer, 0, timestampNS);
break;
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
// I420 Buffer
// Calculate the offsets and lengths for the Y, U, and V planes
int ySize = width * height;
int uvSize = ySize / 4;
// Create YUV Buffers from the YUV data
ByteBuffer yPlane = ByteBuffer.allocateDirect(ySize);
ByteBuffer uPlane = ByteBuffer.allocateDirect(uvSize);
ByteBuffer vPlane = ByteBuffer.allocateDirect(uvSize);
// Copy data from videoBuffer to the respective planes
videoBuffer.position(0);
yPlane.put((ByteBuffer) videoBuffer.slice().limit(ySize));
videoBuffer.position(ySize);
uPlane.put((ByteBuffer) videoBuffer.slice().limit(uvSize));
videoBuffer.position(ySize + uvSize);
vPlane.put((ByteBuffer) videoBuffer.slice().limit(uvSize));
yPlane.rewind(); uPlane.rewind(); vPlane.rewind();
// Create JavaI420Buffer
JavaI420Buffer i420Buffer = JavaI420Buffer.wrap(width, height,
yPlane, width,
uPlane, width / 2,
vPlane, width / 2, null);
videoFrame = new VideoFrame(i420Buffer, 0, timestampNS);
break;
default:
Log.e(TAG, "Color format: " + colorFormat + " is not implemented!!");
return;
}
// Feed the video frame to everyone
for (CapturerObserver obs : observers) {
obs.onFrameCaptured(videoFrame);
}
videoFrame.release();
} catch (Exception e){
e.printStackTrace();
Log.e(TAG, e.getClass().getName() + " occurred, stack trace: " + e.getMessage() + "\n" + e.getCause());
}
}
}
});
// Could create more cases if other drones from DJI require a different approach
Log.d(TAG, "DJICApture");
Log.d(TAG, aircraftModel.getDisplayName());
if (this.aircraftModel.getDisplayName().equals("DJI Air 2S") ||
this.aircraftModel.getDisplayName().equals("DJI Phantom 4 PRO")){
if (aircraftModel.getDisplayName().equals("DJI Air 2S")){
// The Air 2S relies on the VideoDataListener to obtain the video feed
// The onReceive callback provides us the raw H264 (at least according to official documentation). To decode it we send it to our DJICodecManager
// H264 or H265 encoding is done to compress and save bandwidth. (4K video might force a switch to H265 on DJI drones)
app/src/main/java/sq/rogue/rosettadrone/plugins/WebRTC/SocketConnection.java
-96
Ver Arquivo
@@ -1,96 +0,0 @@
package sq.rogue.rosettadrone.plugins.WebRTC;
import android.util.Log;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import okhttp3.WebSocket;
import okhttp3.WebSocketListener;
import okio.ByteString;
// https://github.com/square/okhttp/blob/master/samples/guide/src/main/java/okhttp3/recipes/WebSocketEcho.java
// move to https://gist.github.com/AliYusuf95/557af8be5f360c95fdf029795291eddb
public class SocketConnection extends WebSocketListener {
private static final String TAG = "SocketConnection";
private static SocketConnection instance;
private WebSocket webSocket;
// private WebSocketListener listener;
private OkHttpClient client;
private String serverUrl;
public SocketConnection(String serverUrl) {
this.serverUrl = serverUrl;
this.client = new OkHttpClient();
// this.listener = new DefaultWebSocketListener(); // Initial default listener
connect();
}
public static synchronized SocketConnection getInstance() {
if (instance == null) {
instance = new SocketConnection("ws://192.168.1.220:8090");
}
return instance;
}
private void connect() {
Request request = new Request.Builder().url(serverUrl).build();
webSocket = client.newWebSocket(request, this);
client.dispatcher().executorService().shutdown();
}
// public void setWebSocketListener(WebSocketListener newListener) {
// this.listener = newListener;
// if (webSocket != null) {
// webSocket.close(1000, "Reconnecting with new listener");
// }
// connect();
// }
public WebSocket getWebSocket() {
return webSocket;
}
// private class DefaultWebSocketListener extends WebSocketListener {
@Override
public void onOpen(WebSocket webSocket, Response response) {
Log.d(TAG, "WebSocket connected to " + serverUrl);
}
@Override
public void onMessage(WebSocket webSocket, String text) {
Log.d(TAG, "Received message: " + text);
// Default message handling
}
@Override
public void onMessage(WebSocket webSocket, ByteString bytes) {
Log.d(TAG, "Received bytes: " + bytes.hex());
}
@Override
public void onClosing(WebSocket webSocket, int code, String reason) {
Log.d(TAG, "WebSocket closing: " + reason);
webSocket.close(1000, null);
}
@Override
public void onClosed(WebSocket webSocket, int code, String reason) {
Log.d(TAG, "WebSocket closed: " + reason);
}
@Override
public void onFailure(WebSocket webSocket, Throwable t, Response response) {
Log.e(TAG, "WebSocket error: " + t.getMessage());
}
// }
public void closeConnection() {
if (webSocket != null) {
webSocket.close(1000, "Closing connection");
webSocket = null;
instance = null;
Log.d(TAG, "WebSocket connection closed");
}
}
}
app/src/main/java/sq/rogue/rosettadrone/plugins/WebRTC/WebRTCClient.java
+11 -15
Ver Arquivo
@@ -4,6 +4,7 @@ import static org.webrtc.SessionDescription.Type.OFFER;
import android.content.Context;
import android.util.Log;
import java.util.ArrayList;
import org.json.JSONException;
import org.json.JSONObject;
@@ -22,10 +23,10 @@ import org.webrtc.VideoCapturer;
import org.webrtc.VideoSource;
import org.webrtc.VideoTrack;
import java.util.ArrayList;
import sq.rogue.rosettadrone.plugins.WebRTC.websocket.Socket;
public class WebRTCClient {
private static final String TAG = "WebRTCClient";
private static final String TAG = WebRTCClient.class.getSimpleName();
private final Context context;
// WebRTC related variables
@@ -80,7 +81,7 @@ public class WebRTCClient {
public void handleWebRTCMessage(JSONObject message){
try {
Log.d(TAG, "connectToSignallingServer: got message " + message);
// Log.d(TAG, "connectToSignallingServer: got message " + message);
if (message.getString("type").equals("offer")) {
Log.d(TAG, "connectToSignallingServer: received an offer");
peerConnection.setRemoteDescription(new SimpleSdpObserver(), new SessionDescription(OFFER, message.getString("sdp")));
@@ -92,8 +93,7 @@ public class WebRTCClient {
}
}
catch (JSONException e) {
Log.d(TAG, "Exception during WebRTC message : " + e.getMessage());
// e.printStackTrace();
Log.d(TAG, "JSONException: " + e.getMessage());
}
}
@@ -108,24 +108,22 @@ public class WebRTCClient {
message.put("sdp", sessionDescription.description);
sendMessage(message);
} catch (JSONException e) {
Log.e(TAG, "JSONException " + e.getMessage());
// e.printStackTrace();
Log.e(TAG, "JSONException: " + e.getMessage());
}
}
}, new MediaConstraints());
}
private void sendMessage(Object message) {
Log.d(TAG, "Emitting message to " + peerSocketID);
// Log.d(TAG, "Emitting message to " + peerSocketID);
try {
JSONObject sendMessage = new JSONObject();
sendMessage.put("event", "webrtc_msg");
// append target socketID to sent data.
JSONObject sendMessage = new JSONObject(message.toString());
sendMessage.put("socketID", peerSocketID);
sendMessage.put("data", message);
SocketConnection.getInstance().getWebSocket().send(sendMessage.toString());
Socket.getInstance().send("webrtc_msg", sendMessage);
} catch (JSONException e) {
Log.e(TAG, "JSONException on sendMessage: " + e.getMessage());
Log.e(TAG, "JSONException: " + e.getMessage());
}
}
@@ -202,11 +200,9 @@ public class WebRTCClient {
message.put("id", iceCandidate.sdpMid);
message.put("candidate", iceCandidate.sdp);
Log.d(TAG, "onIceCandidate: sending candidate " + message);
sendMessage(message);
} catch (JSONException e) {
Log.e(TAG, "JSONException " + e.getMessage());
// e.printStackTrace();
}
}
app/src/main/java/sq/rogue/rosettadrone/plugins/WebRTC/websocket/Socket.java
+5 -6
Ver Arquivo
@@ -148,10 +148,10 @@ public class Socket {
JSONObject text = new JSONObject();
text.put("event", event);
text.put("data", new JSONObject(data));
Log.d(TAG, "Try to send data: \n"+ text.toString());
return mRealWebSocket.send(text.toString());
} catch (JSONException e) {
Log.e(TAG, "Try to send data with wrong JSON format");
Log.e(TAG, "JSONException when sending data: " + e.getMessage());
}
}
return false;
@@ -299,7 +299,7 @@ public class Socket {
@Override
public void onMessage(WebSocket webSocket, String text) {
// print received message in log
Log.d(TAG, "New Message received \n" + text);
// Log.d(TAG, "New Message received \n" + text);
// call message listener
postEvent(new SocketEvents.BaseMessageEvent(text));
@@ -317,8 +317,7 @@ public class Socket {
postEvent(new SocketEvents.BaseMessageEvent(event));
} catch (JSONException e) {
// Message text not in JSON format or don't have {event}|{data} object
Log.d(TAG,"Unknown message format.");
Log.d(TAG,"JSONException:" + e.getMessage());
Log.e(TAG,"JSONException:" + e.getMessage());
}
}
@@ -354,7 +353,7 @@ public class Socket {
public void onFailure(WebSocket webSocket, Throwable t, Response response) {
if (!isForceTermination) {
isForceTermination = false; // reset flag
Log.d(TAG, "Socket connection fail, try to reconnect. (" + reconnectionAttempts + ")");
Log.e(TAG, "Socket connection fail, try to reconnect. (" + reconnectionAttempts + ")", t);
changeState(SocketState.CONNECT_ERROR);
reconnect();
}
app/src/main/java/sq/rogue/rosettadrone/plugins/WebRTC/websocket/SocketBuilder.java
+1 -7
Ver Arquivo
@@ -4,7 +4,6 @@ import androidx.annotation.NonNull;
import java.util.concurrent.TimeUnit;
import okhttp3.OkHttpClient;
import okhttp3.Request;
//import okhttp3.logging.HttpLoggingInterceptor;
/**
* Builder class to build websocket connection
@@ -12,13 +11,8 @@ import okhttp3.Request;
public class SocketBuilder {
private Request.Builder request;
// private HttpLoggingInterceptor logging =
// new HttpLoggingInterceptor()
// .setLevel(HttpLoggingInterceptor.Level.HEADERS);
private OkHttpClient.Builder httpClient =
new OkHttpClient.Builder();
// .addInterceptor(logging);
private OkHttpClient.Builder httpClient = new OkHttpClient.Builder();
private SocketBuilder(Request.Builder request) {
this.request = request;
app/src/main/java/sq/rogue/rosettadrone/plugins/WebRTCStreaming.java
+13 -43
Ver Arquivo
@@ -1,43 +1,26 @@
package sq.rogue.rosettadrone.plugins;
//import android.os.Handler;
import android.util.Log;
import java.util.concurrent.TimeUnit;
//import java.io.IOException;
//import java.nio.ByteBuffer;
import dji.common.product.Model;
// import io.socket.client.Socket;
// import sq.rogue.rosettadrone.DroneModel;
//import okhttp3.WebSocket;
import sq.rogue.rosettadrone.Plugin;
import sq.rogue.rosettadrone.RDApplication;
import sq.rogue.rosettadrone.plugins.WebRTC.DJIStreamer;
//import sq.rogue.rosettadrone.plugins.WebRTC.SocketConnection;
import sq.rogue.rosettadrone.plugins.WebRTC.WebRTCMediaOptions;
import sq.rogue.rosettadrone.plugins.WebRTC.websocket.Socket;
import sq.rogue.rosettadrone.plugins.WebRTC.websocket.SocketBuilder;
import sq.rogue.rosettadrone.plugins.WebRTC.websocket.OnStateChangeListener;
import sq.rogue.rosettadrone.plugins.WebRTC.websocket.SocketState;
import sq.rogue.rosettadrone.plugins.WebRTC.websocket.Socket;
import sq.rogue.rosettadrone.plugins.WebRTC.websocket.SocketBuilder;
import sq.rogue.rosettadrone.plugins.WebRTC.websocket.OnStateChangeListener;
import sq.rogue.rosettadrone.plugins.WebRTC.websocket.SocketState;
public class WebRTCStreaming extends Plugin {
private static final String TAG = "WebRTCStreaming";
private static final String TAG = WebRTCStreaming.class.getSimpleName();
private final String WEBSOCKET_URL = "ws://192.168.1.220:8090";
private DJIStreamer djiStreamer;
private Socket mSocket;
private Model aircraftModel;
// WebRTCStreaming.TestSender testSender;
private static final boolean TEST = false; // Send a testing stream
public void start() {
pluginManager.mainActivity.useCustomDecoder = false; // Messes up the buffer received by onYuvDataReceived()
pluginManager.mainActivity.useOutputSurface = false; // Avoid crash when clicking on minimap
// Handler mainHandler = new Handler(pluginManager.mainActivity.getMainLooper());
// Log.e(TAG, "Socket start");
// init websocket
mSocket = SocketBuilder.with(WEBSOCKET_URL)
@@ -68,52 +51,39 @@ public class WebRTCStreaming extends Plugin {
mSocket.connect();
if(TEST || RDApplication.isTestMode) {
// TODO
// testSender = new WebRTCStreaming.TestSender();
// testSender.streamer = this;
// testSender.start();
// TODO: fake video streaming?
}
else {
aircraftModel = pluginManager.mainActivity.mModel.m_model;
if (aircraftModel == null) {
if (pluginManager.mainActivity.mModel.m_model == null) {
String msg = "Couldn't get model. Reconnect or restart app.";
Log.e(TAG, msg);
pluginManager.mainActivity.logMessageDJI(msg);
pluginManager.mainActivity.finish();
return;
}
else {
Log.d(TAG, " djiStreamer start");
djiStreamer = new DJIStreamer(pluginManager.mainActivity, aircraftModel);
Log.d(TAG, " djiStreamer started");
djiStreamer = new DJIStreamer(pluginManager.mainActivity, pluginManager.mainActivity.mModel.m_model);
}
}
}
public void onVideoChange() {
// pluginManager.mainActivity.mCodecManager.enabledYuvData(true);
// pluginManager.mainActivity.mCodecManager.setYuvDataCallback(this);
// TODO: stop/start connections?
Log.d(TAG, "onVideoChange");
}
public void stop() {
if(TEST || RDApplication.isTestMode) {
// TODO
// testSender.stop = true;
// TODO: stop fake video stream;
} else {
Log.d(TAG, "CALLstop");
// pluginManager.mainActivity.mCodecManager.enabledYuvData(false);
// TODO: stop DJIStreamer clients?;
}
if(djiStreamer != null) {
mSocket.close();
}
// TODO: properly close client connections.
mSocket.terminate();
}
public boolean isEnabled() {
return true;
}
}