WebRTC video stream draft

Esse commit está contido em:
Angel Ayala
2024-08-07 23:58:54 -03:00
commit 9d3787719b
10 arquivos alterados com 638 adições e 12 exclusões
+3 -1
Ver Arquivo
@@ -164,7 +164,9 @@ dependencies {
implementation 'javax.mail:mail:1.4.7' // 1.4.7
// exclude 'javax.activation:activation:1.1'
// implementation 'com.sun.mail:android-activation:1.6.6'
// implementation 'io.getstream:stream-webrtc-android:1.1.3'
implementation 'org.webrtc:google-webrtc:1.0.32006'
implementation 'com.squareup.okhttp3:okhttp:4.10.0'
}
+5 -3
Ver Arquivo
@@ -54,7 +54,9 @@
android:label="@string/app_name"
android:roundIcon="@mipmap/ic_launcher_round"
android:supportsRtl="true"
android:theme="@style/AppTheme">
android:theme="@style/AppTheme"
android:usesCleartextTraffic="true"
tools:targetApi="m">
<activity
android:name="sq.rogue.rosettadrone.ConnectionActivity"
@@ -100,8 +102,8 @@
tools:ignore="Instantiatable">
</service>
<service android:name="sq.rogue.rosettadrone.video.VideoService">
</service>
<!--<service android:name="sq.rogue.rosettadrone.video.VideoService">
</service>-->
<!-- DJI SDK needed also by emulator, this is a problem... -->
<uses-library
@@ -4,6 +4,8 @@
*/
package sq.rogue.rosettadrone;
import android.util.Log;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -21,7 +23,8 @@ public class PluginManager {
//List<String> classNames = Arrays.asList("RawVideoStreamer", "AI9Tek");
//List<String> classNames = Arrays.asList("AI9Tek");
List<String> classNames = Arrays.asList();
List<String> classNames = Arrays.asList("WebRTCStreaming");
//List<String> classNames = Arrays.asList();
PluginManager(MainActivity mainActivity) {
this.mainActivity = mainActivity;
@@ -36,7 +39,8 @@ public class PluginManager {
}
} catch (Exception e) {
e.printStackTrace();
// e.printStackTrace();
Log.e(TAG, e.getClass().getName() + " occurred, stack trace: " + e.getMessage() + "\n" + e.getCause());
}
}
}
@@ -0,0 +1,118 @@
package sq.rogue.rosettadrone.plugins.WebRTC;
import android.content.Context;
import android.os.Build;
import android.os.Handler;
import android.util.Log;
import org.json.JSONObject;
import org.webrtc.VideoCapturer;
import java.util.Hashtable;
import dji.common.product.Model;
import okhttp3.WebSocket;
import okhttp3.WebSocketListener;
/**
* The DJIStreamer class will manage all ongoing P2P connections
* with clients, who desire videofeed.
*/
public class DJIStreamer {
private static final String TAG = "DJIStreamer";
private final Context context;
private final Hashtable<String, WebRTCClient> ongoingConnections = new Hashtable<>();
private final Model model;
private WebSocket webSocket;
public DJIStreamer(Context context, Model model){
this.context = context;
this.model = model;
Log.d(TAG, "Pre SocketEvent");
setupSocketEvent();
}
private WebRTCClient getClient(String socketID){
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
return ongoingConnections.getOrDefault(socketID, null);
}
return null;
}
private void removeClient(String socketID){
// TODO: Any other cleanup necessary?.. Let the client stop the VideoCapturer though.
ongoingConnections.remove(socketID);
}
private WebRTCClient addNewClient(String socketID){
VideoCapturer videoCapturer = new DJIVideoCapturer(model);
WebRTCClient client = new WebRTCClient(socketID, context, videoCapturer, new WebRTCMediaOptions());
client.setConnectionChangedListener(new WebRTCClient.PeerConnectionChangedListener() {
@Override
public void onDisconnected() {
removeClient(client.peerSocketID);
Log.d(TAG, "DJIStreamer has removed connection from table. Remaining active sessions: " + ongoingConnections.size());
}
});
ongoingConnections.put(socketID, client);
return client;
}
private void setupSocketEvent(){
SocketConnection socketConnection = SocketConnection.getInstance();
// webSocket = socketConnection.getWebSocket();
// Setting up WebSocket Listener
socketConnection.setWebSocketListener(new WebSocketListener() {
@Override
public void onOpen(WebSocket webSocket, okhttp3.Response response) {
Log.d(TAG, "WebSocket connected");
}
@Override
public void onMessage(WebSocket webSocket, String text) {
Handler mainHandler = new Handler(context.getMainLooper());
Runnable myRunnable = () -> {
try {
JSONObject messageJson = new JSONObject(text);
String peerSocketID = messageJson.getString("socketID");
Log.d(TAG, "Received WebRTCMessage: " + peerSocketID);
WebRTCClient client = getClient(peerSocketID);
if (client == null){
// A new client wants to establish a P2P
client = addNewClient(peerSocketID);
}
// Then just pass the message to the client
client.handleWebRTCMessage(messageJson.getJSONObject("message"));
} catch (Exception e) {
Log.e(TAG, "Error processing WebRTC message", e);
}
};
mainHandler.post(myRunnable);
}
@Override
public void onClosing(WebSocket webSocket, int code, String reason) {
Log.d(TAG, "WebSocket closing: " + reason);
webSocket.close(1000, null);
}
@Override
public void onClosed(WebSocket webSocket, int code, String reason) {
Log.d(TAG, "WebSocket closed: " + reason);
}
@Override
public void onFailure(WebSocket webSocket, Throwable t, okhttp3.Response response) {
Log.e(TAG, "WebSocket error: " + t.getMessage());
}
});
Log.d(TAG, "Socket instantiated");
}
}
@@ -0,0 +1,133 @@
/* Originally sourced from
* https://chromium.googlesource.com/external/webrtc/+/b6760f9e4442410f2bcb6090b3b89bf709e2fce2/webrtc/api/android/java/src/org/webrtc/CameraVideoCapturer.java
* and rewritten to work for DJI drones.
* */
package sq.rogue.rosettadrone.plugins.WebRTC;
import org.webrtc.CapturerObserver;
import org.webrtc.NV12Buffer;
import org.webrtc.SurfaceTextureHelper;
import org.webrtc.VideoCapturer;
import org.webrtc.VideoFrame;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.media.MediaFormat;
import android.os.SystemClock;
import android.util.Log;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.concurrent.TimeUnit;
import dji.common.product.Model;
import dji.sdk.camera.VideoFeeder;
import dji.sdk.codec.DJICodecManager;
public class DJIVideoCapturer implements VideoCapturer {
private final static String TAG = "DJIStreamer";
private static DJICodecManager codecManager;
private static final ArrayList<CapturerObserver> observers = new ArrayList<CapturerObserver>();
private final Model aircraftModel;
private Context context;
private CapturerObserver capturerObserver;
public DJIVideoCapturer(Model aircraftModel){
this.aircraftModel = aircraftModel;
}
private void setupVideoListener(){
Log.d(TAG, "setupVideoListener");
if(codecManager != null) {
Log.d(TAG, "codecManager not null");
return;
}
// Pass SurfaceTexture as null to force the Yuv callback - width and height for the surface texture does not matter
codecManager = new DJICodecManager(context, (SurfaceTexture)null, 0, 0);
codecManager.enabledYuvData(true);
codecManager.setYuvDataCallback(new DJICodecManager.YuvDataCallback() {
@Override
public void onYuvDataReceived(MediaFormat mediaFormat, ByteBuffer videoBuffer, int dataSize, int width, int height) {
if (videoBuffer != null){
try{
// TODO: We need to check which color format they are using by doing a lookup in our MediaFormat, otherwise we get green artifacts
// This can change with Android/device versions. The format might actually change, seemingly at random, according to community reports...
// Other possible buffers we might have to use: I420Buffer
long timestampNS = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime());
NV12Buffer buffer = new NV12Buffer(width,
height,
mediaFormat.getInteger(MediaFormat.KEY_STRIDE),
mediaFormat.getInteger(MediaFormat.KEY_SLICE_HEIGHT),
videoBuffer,
null);
VideoFrame videoFrame = new VideoFrame(buffer, 0, timestampNS);
// Feed the video frame to everyone
for (CapturerObserver obs : observers) {
obs.onFrameCaptured(videoFrame);
}
videoFrame.release();
} catch (Exception e){
e.printStackTrace();
}
}
}
});
// Could create more cases if other drones from DJI require a different approach
Log.d(TAG, "DJICApture");
Log.d(TAG, aircraftModel.getDisplayName());
if (this.aircraftModel.getDisplayName().equals("DJI Air 2S") ||
this.aircraftModel.getDisplayName().equals("DJI Phantom 4 PRO")){
// The Air 2S relies on the VideoDataListener to obtain the video feed
// The onReceive callback provides us the raw H264 (at least according to official documentation). To decode it we send it to our DJICodecManager
// H264 or H265 encoding is done to compress and save bandwidth. (4K video might force a switch to H265 on DJI drones)
VideoFeeder.VideoDataListener videoDataListener = new VideoFeeder.VideoDataListener() {
@Override
public void onReceive(byte[] bytes, int dataSize) {
// Pass the encoded data along to obtain the YUV-color data
codecManager.sendDataToDecoder(bytes, dataSize);
}
};
VideoFeeder.getInstance().getPrimaryVideoFeed().addVideoDataListener(videoDataListener);
}
}
@Override
public void initialize(SurfaceTextureHelper surfaceTextureHelper, Context applicationContext,
CapturerObserver capturerObserver) {
this.context = applicationContext;
this.capturerObserver = capturerObserver;
observers.add(capturerObserver);
}
@Override
public void startCapture(int width, int height, int framerate) {
// Hook onto the DJI onYuvDataReceived event
setupVideoListener();
}
@Override
public void stopCapture() throws InterruptedException {
}
@Override
public void changeCaptureFormat(int width, int height, int framerate) {
// Empty on purpose
}
@Override
public void dispose() {
// Stop receiving frames on the callback from the decoder
if (observers.contains(capturerObserver))
observers.remove(capturerObserver);
}
@Override
public boolean isScreencast() {
return false;
}
}
@@ -0,0 +1,24 @@
package sq.rogue.rosettadrone.plugins.WebRTC;
import org.webrtc.SdpObserver;
import org.webrtc.SessionDescription;
class SimpleSdpObserver implements SdpObserver {
@Override
public void onCreateSuccess(SessionDescription sessionDescription) {
}
@Override
public void onSetSuccess() {
}
@Override
public void onCreateFailure(String s) {
}
@Override
public void onSetFailure(String s) {
}
}
@@ -0,0 +1,93 @@
package sq.rogue.rosettadrone.plugins.WebRTC;
import android.util.Log;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import okhttp3.WebSocket;
import okhttp3.WebSocketListener;
import okio.ByteString;
// https://github.com/square/okhttp/blob/master/samples/guide/src/main/java/okhttp3/recipes/WebSocketEcho.java
public class SocketConnection {
private static final String TAG = "SocketConnection";
private static SocketConnection instance;
private WebSocket webSocket;
private WebSocketListener listener;
private OkHttpClient client;
private String serverUrl;
public SocketConnection(String serverUrl) {
this.serverUrl = serverUrl;
this.client = new OkHttpClient();
this.listener = new DefaultWebSocketListener(); // Initial default listener
connect();
}
public static synchronized SocketConnection getInstance() {
if (instance == null) {
instance = new SocketConnection("ws://192.168.1.220:8090");
}
return instance;
}
private void connect() {
Request request = new Request.Builder().url(serverUrl).build();
webSocket = client.newWebSocket(request, listener);
client.dispatcher().executorService().shutdown();
}
public void setWebSocketListener(WebSocketListener newListener) {
this.listener = newListener;
if (webSocket != null) {
webSocket.close(1000, "Reconnecting with new listener");
}
connect();
}
public WebSocket getWebSocket() {
return webSocket;
}
private class DefaultWebSocketListener extends WebSocketListener {
@Override
public void onOpen(WebSocket webSocket, Response response) {
Log.d(TAG, "WebSocket connected to " + serverUrl);
}
@Override
public void onMessage(WebSocket webSocket, String text) {
Log.d(TAG, "Received message: " + text);
// Default message handling
}
@Override
public void onMessage(WebSocket webSocket, ByteString bytes) {
Log.d(TAG, "Received bytes: " + bytes.hex());
}
@Override
public void onClosing(WebSocket webSocket, int code, String reason) {
Log.d(TAG, "WebSocket closing: " + reason);
webSocket.close(1000, null);
}
@Override
public void onClosed(WebSocket webSocket, int code, String reason) {
Log.d(TAG, "WebSocket closed: " + reason);
}
@Override
public void onFailure(WebSocket webSocket, Throwable t, Response response) {
Log.e(TAG, "WebSocket error: " + t.getMessage());
}
}
public void closeConnection() {
if (webSocket != null) {
webSocket.close(1000, "Closing connection");
webSocket = null;
instance = null;
Log.d(TAG, "WebSocket connection closed");
}
}
}
@@ -0,0 +1,238 @@
package sq.rogue.rosettadrone.plugins.WebRTC;
import static org.webrtc.SessionDescription.Type.OFFER;
import android.content.Context;
import android.util.Log;
import org.json.JSONException;
import org.json.JSONObject;
import org.webrtc.DataChannel;
import org.webrtc.DefaultVideoDecoderFactory;
import org.webrtc.DefaultVideoEncoderFactory;
import org.webrtc.EglBase;
import org.webrtc.IceCandidate;
import org.webrtc.MediaConstraints;
import org.webrtc.MediaStream;
import org.webrtc.PeerConnection;
import org.webrtc.PeerConnectionFactory;
import org.webrtc.RtpReceiver;
import org.webrtc.SessionDescription;
import org.webrtc.VideoCapturer;
import org.webrtc.VideoSource;
import org.webrtc.VideoTrack;
import java.util.ArrayList;
public class WebRTCClient {
private static final String TAG = "WebRTCClient";
private final Context context;
// WebRTC related variables
private PeerConnection peerConnection;
private VideoTrack videoTrackFromCamera;
private final WebRTCMediaOptions options;
private final VideoCapturer videoCapturer;
private PeerConnectionChangedListener connectionChangedListener;
public void setConnectionChangedListener(PeerConnectionChangedListener connectionChangedListener) { this.connectionChangedListener = connectionChangedListener; }
// Peer variables of the client requesting a stream
public final String peerSocketID;
private static PeerConnectionFactory factory;
private static PeerConnectionFactory getFactory(Context context){
if (factory == null) {
initializeFactory(context);
}
return factory;
}
public WebRTCClient(String peerSocketID, Context context, VideoCapturer videoCapturer, WebRTCMediaOptions options){
this.peerSocketID = peerSocketID;
this.context = context;
this.options = options;
this.videoCapturer = videoCapturer;
createVideoTrackFromVideoCapturer();
initializePeerConnection();
startStreamingVideo();
}
private static void initializeFactory(Context context){
// EglBase seems to be used for Hardware-acceleration for our video. Could help with smoothing things. (keep it)
EglBase rootEglBase = EglBase.create();
// Initialize the PeerConnectionFactory
PeerConnectionFactory.InitializationOptions options = PeerConnectionFactory.InitializationOptions.builder(context)
.setEnableInternalTracer(true)
.setFieldTrials("WebRTC-H264HighProfile/Enabled/")
.createInitializationOptions();
PeerConnectionFactory.initialize(options);
// Now configure and build the factory
factory = PeerConnectionFactory
.builder()
.setVideoDecoderFactory(new DefaultVideoDecoderFactory(rootEglBase.getEglBaseContext()))
.setVideoEncoderFactory(new DefaultVideoEncoderFactory(rootEglBase.getEglBaseContext(), true, true))
.setOptions(new PeerConnectionFactory.Options()).createPeerConnectionFactory();
}
public void handleWebRTCMessage(JSONObject message){
try {
Log.d(TAG, "connectToSignallingServer: got message " + message);
if (message.getString("type").equals("offer")) {
Log.d(TAG, "connectToSignallingServer: received an offer");
peerConnection.setRemoteDescription(new SimpleSdpObserver(), new SessionDescription(OFFER, message.getString("sdp")));
answerCall();
} else if (message.getString("type").equals("candidate")) {
Log.d(TAG, "connectToSignallingServer: receiving candidates");
IceCandidate candidate = new IceCandidate(message.getString("id"), message.getInt("label"), message.getString("candidate"));
peerConnection.addIceCandidate(candidate);
}
}
catch (JSONException e) {
Log.d(TAG, "Exception with socket : " + e.getMessage());
e.printStackTrace();
}
}
private void answerCall() {
peerConnection.createAnswer(new SimpleSdpObserver() {
@Override
public void onCreateSuccess(SessionDescription sessionDescription) {
peerConnection.setLocalDescription(new SimpleSdpObserver(), sessionDescription);
JSONObject message = new JSONObject();
try {
message.put("type", "answer");
message.put("sdp", sessionDescription.description);
sendMessage(message);
} catch (JSONException e) {
e.printStackTrace();
}
}
}, new MediaConstraints());
}
private void sendMessage(Object message) {
Log.d(TAG, "Emitting message to " + peerSocketID);
SocketConnection.getInstance().getWebSocket().send(message.toString());
// // Ensure WebSocketWrapper is initialized and available
// WebSocketWrapper webSocketWrapper = new WebSocketWrapper("ws://192.168.1.220:8090");
// WebSocket webSocket = SocketConnection.getInstance().getWebSocket();
// if (webSocket != null) {
// webSocket.send(message.toString());
// } else {
// Log.e(TAG, "WebSocket is not available");
// }
}
private void createVideoTrackFromVideoCapturer() {
VideoSource videoSource = getFactory(context).createVideoSource(false);
// Instantiate our custom video capturer to get video from our drone
videoCapturer.initialize(null, context, videoSource.getCapturerObserver());
videoCapturer.startCapture(options.VIDEO_RESOLUTION_WIDTH, options.VIDEO_RESOLUTION_HEIGHT, options.FPS);
videoTrackFromCamera = getFactory(context).createVideoTrack(options.VIDEO_SOURCE_ID, videoSource);
videoTrackFromCamera.setEnabled(true);
}
private void initializePeerConnection() {
peerConnection = createPeerConnection();
}
private void startStreamingVideo() {
MediaStream mediaStream = getFactory(context).createLocalMediaStream(options.MEDIA_STREAM_ID);
mediaStream.addTrack(videoTrackFromCamera);
peerConnection.addStream(mediaStream);
}
private PeerConnection createPeerConnection() {
ArrayList<PeerConnection.IceServer> iceServers = new ArrayList<>();
PeerConnection.IceServer stun = PeerConnection.IceServer.builder("stun:stun.l.google.com:19302").createIceServer();
iceServers.add(stun);
PeerConnection.RTCConfiguration rtcConfig = new PeerConnection.RTCConfiguration(iceServers);
PeerConnection.Observer pcObserver = new PeerConnection.Observer() {
@Override
public void onSignalingChange(PeerConnection.SignalingState signalingState) {
Log.d(TAG, "onSignalingChange: ");
}
@Override
public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
switch (iceConnectionState){
case DISCONNECTED:
Log.d(TAG, "PEER HAS DISCONNECTED");
if (connectionChangedListener != null)
connectionChangedListener.onDisconnected();
// Dispose of the capturer and then the peer connection to clean up properly
videoCapturer.dispose();
break;
}
}
@Override
public void onIceConnectionReceivingChange(boolean b) {
Log.d(TAG, "onIceConnectionReceivingChange: ");
}
@Override
public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
Log.d(TAG, "onIceGatheringChange: ");
}
@Override
public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
// We are not interested in displaying whatever video feed we receive from the other end.
// Not that we are getting any..
}
@Override
public void onIceCandidate(IceCandidate iceCandidate) {
Log.d(TAG, "onIceCandidate: ");
JSONObject message = new JSONObject();
try {
message.put("type", "candidate");
message.put("label", iceCandidate.sdpMLineIndex);
message.put("id", iceCandidate.sdpMid);
message.put("candidate", iceCandidate.sdp);
Log.d(TAG, "onIceCandidate: sending candidate " + message);
sendMessage(message);
} catch (JSONException e) {
e.printStackTrace();
}
}
@Override
public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) { Log.d(TAG, "onIceCandidatesRemoved: "); }
@Override
public void onAddStream(MediaStream mediaStream) { Log.d(TAG, "onAddStream: "); }
@Override
public void onRemoveStream(MediaStream mediaStream) {
Log.d(TAG, "onRemoveStream: ");
}
@Override
public void onDataChannel(DataChannel dataChannel) {
Log.d(TAG, "onDataChannel: ");
}
@Override
public void onRenegotiationNeeded() {
Log.d(TAG, "onRenegotiationNeeded: ");
}
};
return getFactory(context).createPeerConnection(rtcConfig, pcObserver);
}
public interface PeerConnectionChangedListener {
void onDisconnected(); // Is called when our peer disconnects from the call
}
}
@@ -0,0 +1,9 @@
package sq.rogue.rosettadrone.plugins.WebRTC;
public class WebRTCMediaOptions {
String MEDIA_STREAM_ID = "Phantom4Pro";
String VIDEO_SOURCE_ID = "Phantom4Prov0";
int VIDEO_RESOLUTION_WIDTH = 320;
int VIDEO_RESOLUTION_HEIGHT = 240;
int FPS = 30;
}
@@ -2,18 +2,19 @@ package sq.rogue.rosettadrone.plugins;
import android.util.Log;
import java.io.IOException;
import java.nio.ByteBuffer;
//import java.io.IOException;
//import java.nio.ByteBuffer;
import dji.common.product.Model;
// import io.socket.client.Socket;
import sq.rogue.rosettadrone.DroneModel;
// import sq.rogue.rosettadrone.DroneModel;
import sq.rogue.rosettadrone.Plugin;
import sq.rogue.rosettadrone.RDApplication;
//import sq.rogue.rosettadrone.plugins.WebRTC.DJIStreamer;
//import sq.rogue.rosettadrone.plugins.WebRTC.SocketConnection;
import sq.rogue.rosettadrone.plugins.WebRTC.SocketConnection;
public class WebRTCStreaming extends Plugin {
private static final String TAG = "WebRTCStreaming";
@@ -59,7 +60,9 @@ public class WebRTCStreaming extends Plugin {
public void start() {
pluginManager.mainActivity.useCustomDecoder = false; // Messes up the buffer received by onYuvDataReceived()
pluginManager.mainActivity.useOutputSurface = false; // Avoid crash when clicking on minimap
// socket = SocketConnection.getInstance();
Log.e(TAG, "SocketConnection() call");
SocketConnection socket = SocketConnection.getInstance();
Log.e(TAG, "start() call");
if(TEST || RDApplication.isTestMode) {
// TODO
@@ -106,7 +109,7 @@ public class WebRTCStreaming extends Plugin {
}
public boolean isEnabled() {
return false;
return true;
}